2024-11-15 00:48:11,335 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 00:48:11,351 main DEBUG Took 0.012611 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-15 00:48:11,352 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-15 00:48:11,352 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-15 00:48:11,354 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-15 00:48:11,355 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,365 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-15 00:48:11,381 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,382 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,383 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,384 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,385 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,385 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,387 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,387 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,388 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,389 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,390 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,390 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,391 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,392 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,393 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,393 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,394 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,394 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,396 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,396 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,397 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,397 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,398 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,398 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-15 00:48:11,398 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,399 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-15 00:48:11,400 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-15 00:48:11,401 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-15 00:48:11,403 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-15 00:48:11,404 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-15 00:48:11,405 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-15 00:48:11,405 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-15 00:48:11,444 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-15 00:48:11,446 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-15 00:48:11,448 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-15 00:48:11,449 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-15 00:48:11,449 main DEBUG createAppenders(={Console}) 2024-11-15 00:48:11,450 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-15 00:48:11,450 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-15 00:48:11,450 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-15 00:48:11,451 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-15 00:48:11,451 main DEBUG OutputStream closed 2024-11-15 00:48:11,451 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-15 00:48:11,452 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-15 00:48:11,452 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-15 00:48:11,622 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-15 00:48:11,626 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-15 00:48:11,628 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-15 00:48:11,630 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-15 00:48:11,631 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-15 00:48:11,632 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-15 00:48:11,632 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-15 00:48:11,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-15 00:48:11,633 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-15 00:48:11,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-15 00:48:11,634 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-15 00:48:11,635 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-15 00:48:11,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-15 00:48:11,636 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-15 00:48:11,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-15 00:48:11,637 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-15 00:48:11,638 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-15 00:48:11,639 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-15 00:48:11,642 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15 00:48:11,643 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-15 00:48:11,643 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-15 00:48:11,644 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-15T00:48:12,037 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb 2024-11-15 00:48:12,040 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-15 00:48:12,041 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-15T00:48:12,050 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-15T00:48:12,099 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-15T00:48:12,117 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/cluster_4d500b26-95f0-bc79-8ccc-862558111e78, deleteOnExit=true 2024-11-15T00:48:12,117 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-15T00:48:12,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/test.cache.data in system properties and HBase conf 2024-11-15T00:48:12,119 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/hadoop.tmp.dir in system properties and HBase conf 2024-11-15T00:48:12,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/hadoop.log.dir in system properties and HBase conf 2024-11-15T00:48:12,120 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-15T00:48:12,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-15T00:48:12,121 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-15T00:48:12,225 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-15T00:48:12,449 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-15T00:48:12,454 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-15T00:48:12,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-15T00:48:12,455 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-15T00:48:12,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T00:48:12,456 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-15T00:48:12,457 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-15T00:48:12,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-15T00:48:12,458 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T00:48:12,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-15T00:48:12,459 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/nfs.dump.dir in system properties and HBase conf 2024-11-15T00:48:12,460 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/java.io.tmpdir in system properties and HBase conf 2024-11-15T00:48:12,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-15T00:48:12,461 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-15T00:48:12,462 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-15T00:48:13,667 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-15T00:48:13,775 INFO [Time-limited test {}] log.Log(170): Logging initialized @3413ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-15T00:48:13,865 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T00:48:13,951 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T00:48:13,983 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T00:48:13,983 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T00:48:13,985 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-15T00:48:14,000 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T00:48:14,004 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/hadoop.log.dir/,AVAILABLE} 2024-11-15T00:48:14,005 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T00:48:14,246 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/java.io.tmpdir/jetty-localhost-36611-hadoop-hdfs-3_4_1-tests_jar-_-any-18407573441836199341/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T00:48:14,277 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:36611} 2024-11-15T00:48:14,277 INFO [Time-limited test {}] server.Server(415): Started @3916ms 2024-11-15T00:48:15,016 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-15T00:48:15,030 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-15T00:48:15,056 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-15T00:48:15,057 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-15T00:48:15,057 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 600000ms 2024-11-15T00:48:15,069 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/hadoop.log.dir/,AVAILABLE} 2024-11-15T00:48:15,070 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-15T00:48:15,193 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d1d0948{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/java.io.tmpdir/jetty-localhost-46687-hadoop-hdfs-3_4_1-tests_jar-_-any-17987782900050434445/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T00:48:15,193 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:46687} 2024-11-15T00:48:15,194 INFO [Time-limited test {}] server.Server(415): Started @4833ms 2024-11-15T00:48:15,266 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-15T00:48:16,706 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/cluster_4d500b26-95f0-bc79-8ccc-862558111e78/data/data2/current/BP-1791232400-172.17.0.3-1731631693107/current, will proceed with Du for space computation calculation, 2024-11-15T00:48:16,706 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/cluster_4d500b26-95f0-bc79-8ccc-862558111e78/data/data1/current/BP-1791232400-172.17.0.3-1731631693107/current, will proceed with Du for space computation calculation, 2024-11-15T00:48:16,738 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-15T00:48:16,790 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb 2024-11-15T00:48:16,793 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8aa1673f49f557b2 with lease ID 0x73a1f456e777a23: Processing first storage report for DS-9a544359-9c18-4087-b458-d3b1e86c3abe from datanode DatanodeRegistration(127.0.0.1:37847, datanodeUuid=bfcb1796-17ac-4b3f-8ab9-7fe5e77a4458, infoPort=36083, infoSecurePort=0, ipcPort=38165, storageInfo=lv=-57;cid=testClusterID;nsid=1548114726;c=1731631693107) 2024-11-15T00:48:16,794 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8aa1673f49f557b2 with lease ID 0x73a1f456e777a23: from storage DS-9a544359-9c18-4087-b458-d3b1e86c3abe node DatanodeRegistration(127.0.0.1:37847, datanodeUuid=bfcb1796-17ac-4b3f-8ab9-7fe5e77a4458, infoPort=36083, infoSecurePort=0, ipcPort=38165, storageInfo=lv=-57;cid=testClusterID;nsid=1548114726;c=1731631693107), blocks: 0, hasStaleStorage: true, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T00:48:16,795 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0x8aa1673f49f557b2 with lease ID 0x73a1f456e777a23: Processing first storage report for DS-b3bd233b-85cd-4ca5-927f-c051bf558c4c from datanode DatanodeRegistration(127.0.0.1:37847, datanodeUuid=bfcb1796-17ac-4b3f-8ab9-7fe5e77a4458, infoPort=36083, infoSecurePort=0, ipcPort=38165, storageInfo=lv=-57;cid=testClusterID;nsid=1548114726;c=1731631693107) 2024-11-15T00:48:16,795 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0x8aa1673f49f557b2 with lease ID 0x73a1f456e777a23: from storage DS-b3bd233b-85cd-4ca5-927f-c051bf558c4c node DatanodeRegistration(127.0.0.1:37847, datanodeUuid=bfcb1796-17ac-4b3f-8ab9-7fe5e77a4458, infoPort=36083, infoSecurePort=0, ipcPort=38165, storageInfo=lv=-57;cid=testClusterID;nsid=1548114726;c=1731631693107), blocks: 0, hasStaleStorage: false, processing time: 1 msecs, invalidatedBlocks: 0 2024-11-15T00:48:16,867 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/cluster_4d500b26-95f0-bc79-8ccc-862558111e78/zookeeper_0, clientPort=59615, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/cluster_4d500b26-95f0-bc79-8ccc-862558111e78/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/cluster_4d500b26-95f0-bc79-8ccc-862558111e78/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-15T00:48:16,876 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=59615 2024-11-15T00:48:16,890 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:48:16,897 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:48:17,186 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741825_1001 (size=7) 2024-11-15T00:48:17,590 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd with version=8 2024-11-15T00:48:17,591 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/hbase-staging 2024-11-15T00:48:17,681 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-15T00:48:17,925 INFO [Time-limited test {}] client.ConnectionUtils(128): master/fb2f6cb7846a:0 server-side Connection retries=45 2024-11-15T00:48:17,935 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:48:17,935 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T00:48:17,941 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T00:48:17,942 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:48:17,942 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T00:48:18,133 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-15T00:48:18,200 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-15T00:48:18,213 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-15T00:48:18,218 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T00:48:18,248 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 20984 (auto-detected) 2024-11-15T00:48:18,249 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:03 (auto-detected) 2024-11-15T00:48:18,269 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:46397 2024-11-15T00:48:18,289 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:46397 connecting to ZooKeeper ensemble=127.0.0.1:59615 2024-11-15T00:48:18,415 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:463970x0, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T00:48:18,418 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:46397-0x1013bf6c77b0000 connected 2024-11-15T00:48:18,581 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:48:18,584 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:48:18,596 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:48:18,600 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd, hbase.cluster.distributed=false 2024-11-15T00:48:18,626 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T00:48:18,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=46397 2024-11-15T00:48:18,632 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=46397 2024-11-15T00:48:18,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=46397 2024-11-15T00:48:18,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=46397 2024-11-15T00:48:18,633 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=46397 2024-11-15T00:48:18,757 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/fb2f6cb7846a:0 server-side Connection retries=45 2024-11-15T00:48:18,760 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:48:18,760 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-15T00:48:18,760 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-15T00:48:18,760 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-15T00:48:18,761 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-15T00:48:18,764 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-15T00:48:18,768 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-15T00:48:18,769 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.3:42925 2024-11-15T00:48:18,772 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:42925 connecting to ZooKeeper ensemble=127.0.0.1:59615 2024-11-15T00:48:18,773 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:48:18,776 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:48:18,793 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:429250x0, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-15T00:48:18,794 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:429250x0, quorum=127.0.0.1:59615, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:48:18,794 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:42925-0x1013bf6c77b0001 connected 2024-11-15T00:48:18,798 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-15T00:48:18,808 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-15T00:48:18,811 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-15T00:48:18,818 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-15T00:48:18,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=42925 2024-11-15T00:48:18,818 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=42925 2024-11-15T00:48:18,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=42925 2024-11-15T00:48:18,819 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=42925 2024-11-15T00:48:18,820 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=42925 2024-11-15T00:48:18,836 DEBUG [M:0;fb2f6cb7846a:46397 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;fb2f6cb7846a:46397 2024-11-15T00:48:18,837 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:18,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:48:18,849 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:48:18,852 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:18,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:18,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-15T00:48:18,880 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:18,881 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-15T00:48:18,883 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/fb2f6cb7846a,46397,1731631697729 from backup master directory 2024-11-15T00:48:18,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:48:18,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:18,967 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-15T00:48:18,969 WARN [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T00:48:18,970 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:18,973 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-15T00:48:18,975 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-15T00:48:19,035 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/hbase.id] with ID: 4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5 2024-11-15T00:48:19,036 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/.tmp/hbase.id 2024-11-15T00:48:19,049 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741826_1002 (size=42) 2024-11-15T00:48:19,051 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/.tmp/hbase.id]:[hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/hbase.id] 2024-11-15T00:48:19,107 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-15T00:48:19,114 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-15T00:48:19,144 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 27ms. 2024-11-15T00:48:19,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:19,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:19,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741827_1003 (size=196) 2024-11-15T00:48:19,616 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:48:19,621 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-15T00:48:19,641 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:19,648 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T00:48:19,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741828_1004 (size=1189) 2024-11-15T00:48:20,110 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store 2024-11-15T00:48:20,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741829_1005 (size=34) 2024-11-15T00:48:20,138 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-15T00:48:20,141 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:48:20,143 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T00:48:20,143 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:48:20,143 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:48:20,146 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T00:48:20,146 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:48:20,146 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:48:20,148 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731631700143Disabling compacts and flushes for region at 1731631700143Disabling writes for close at 1731631700146 (+3 ms)Writing region close event to WAL at 1731631700146Closed at 1731631700146 2024-11-15T00:48:20,151 WARN [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/.initializing 2024-11-15T00:48:20,151 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/WALs/fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:20,161 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T00:48:20,179 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fb2f6cb7846a%2C46397%2C1731631697729, suffix=, logDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/WALs/fb2f6cb7846a,46397,1731631697729, archiveDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/oldWALs, maxLogs=10 2024-11-15T00:48:20,207 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/WALs/fb2f6cb7846a,46397,1731631697729/fb2f6cb7846a%2C46397%2C1731631697729.1731631700185, exclude list is [], retry=0 2024-11-15T00:48:20,246 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37847,DS-9a544359-9c18-4087-b458-d3b1e86c3abe,DISK] 2024-11-15T00:48:20,250 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-15T00:48:20,292 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/WALs/fb2f6cb7846a,46397,1731631697729/fb2f6cb7846a%2C46397%2C1731631697729.1731631700185 2024-11-15T00:48:20,293 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36083:36083)] 2024-11-15T00:48:20,293 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:48:20,294 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:48:20,297 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,298 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,334 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,359 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-15T00:48:20,362 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:20,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:48:20,365 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,369 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-15T00:48:20,369 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:20,370 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:20,371 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,374 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-15T00:48:20,374 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:20,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:20,375 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,379 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-15T00:48:20,379 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:20,381 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:20,381 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,385 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,387 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,393 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,394 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,397 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-15T00:48:20,402 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-15T00:48:20,407 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:48:20,409 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68736575, jitterRate=0.024254783987998962}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-15T00:48:20,416 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1731631700310Initializing all the Stores at 1731631700312 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631700313 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631700314 (+1 ms)Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631700314Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631700314Cleaning up temporary data from old regions at 1731631700394 (+80 ms)Region opened successfully at 1731631700416 (+22 ms) 2024-11-15T00:48:20,417 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-15T00:48:20,452 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c803cde, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fb2f6cb7846a/172.17.0.3:0 2024-11-15T00:48:20,485 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-15T00:48:20,496 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-15T00:48:20,496 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-15T00:48:20,499 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-15T00:48:20,500 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 1 msec 2024-11-15T00:48:20,506 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 5 msec 2024-11-15T00:48:20,506 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-15T00:48:20,531 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-15T00:48:20,543 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-15T00:48:20,583 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-15T00:48:20,586 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-15T00:48:20,590 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-15T00:48:20,615 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-15T00:48:20,619 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-15T00:48:20,627 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-15T00:48:20,638 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-15T00:48:20,641 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-15T00:48:20,648 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-15T00:48:20,675 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-15T00:48:20,688 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-15T00:48:20,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T00:48:20,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:20,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-15T00:48:20,701 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:20,705 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=fb2f6cb7846a,46397,1731631697729, sessionid=0x1013bf6c77b0000, setting cluster-up flag (Was=false) 2024-11-15T00:48:20,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:20,732 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:20,764 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-15T00:48:20,768 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:20,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:20,795 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:20,827 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-15T00:48:20,831 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:20,842 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-15T00:48:20,923 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-15T00:48:20,928 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(746): ClusterId : 4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5 2024-11-15T00:48:20,931 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-15T00:48:20,936 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-15T00:48:20,945 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-15T00:48:20,947 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-15T00:48:20,947 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-15T00:48:20,951 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: fb2f6cb7846a,46397,1731631697729 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-15T00:48:20,954 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-15T00:48:20,955 DEBUG [RS:0;fb2f6cb7846a:42925 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f2f79d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=fb2f6cb7846a/172.17.0.3:0 2024-11-15T00:48:20,958 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:48:20,958 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:48:20,958 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:48:20,958 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=5, maxPoolSize=5 2024-11-15T00:48:20,958 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/fb2f6cb7846a:0, corePoolSize=10, maxPoolSize=10 2024-11-15T00:48:20,959 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:20,959 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=2, maxPoolSize=2 2024-11-15T00:48:20,959 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:20,961 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1731631730960 2024-11-15T00:48:20,963 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-15T00:48:20,964 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-15T00:48:20,965 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T00:48:20,965 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-15T00:48:20,968 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-15T00:48:20,969 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-15T00:48:20,969 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-15T00:48:20,969 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-15T00:48:20,971 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:20,971 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;fb2f6cb7846a:42925 2024-11-15T00:48:20,971 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-15T00:48:20,970 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:20,974 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-15T00:48:20,975 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-15T00:48:20,975 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-15T00:48:20,976 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-15T00:48:20,976 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-15T00:48:20,976 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-15T00:48:20,978 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-15T00:48:20,979 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(2659): reportForDuty to master=fb2f6cb7846a,46397,1731631697729 with port=42925, startcode=1731631698709 2024-11-15T00:48:20,979 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-15T00:48:20,981 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.large.0-1731631700980,5,FailOnTimeoutGroup] 2024-11-15T00:48:20,981 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.small.0-1731631700981,5,FailOnTimeoutGroup] 2024-11-15T00:48:20,982 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:20,982 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-15T00:48:20,983 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:20,984 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:20,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741831_1007 (size=1321) 2024-11-15T00:48:20,993 DEBUG [RS:0;fb2f6cb7846a:42925 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-15T00:48:21,026 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:46195, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-15T00:48:21,031 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46397 {}] master.ServerManager(363): Checking decommissioned status of RegionServer fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:21,033 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46397 {}] master.ServerManager(517): Registering regionserver=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:21,046 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd 2024-11-15T00:48:21,047 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:34051 2024-11-15T00:48:21,047 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-15T00:48:21,135 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T00:48:21,137 DEBUG [RS:0;fb2f6cb7846a:42925 {}] zookeeper.ZKUtil(111): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:21,137 WARN [RS:0;fb2f6cb7846a:42925 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-15T00:48:21,138 INFO [RS:0;fb2f6cb7846a:42925 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T00:48:21,138 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/WALs/fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:21,142 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [fb2f6cb7846a,42925,1731631698709] 2024-11-15T00:48:21,166 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-15T00:48:21,187 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-15T00:48:21,193 INFO [RS:0;fb2f6cb7846a:42925 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-15T00:48:21,193 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:21,194 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-15T00:48:21,199 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-15T00:48:21,201 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:21,201 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,201 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,201 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,201 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,202 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,202 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/fb2f6cb7846a:0, corePoolSize=2, maxPoolSize=2 2024-11-15T00:48:21,203 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,203 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,203 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,203 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,203 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,204 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/fb2f6cb7846a:0, corePoolSize=1, maxPoolSize=1 2024-11-15T00:48:21,204 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/fb2f6cb7846a:0, corePoolSize=3, maxPoolSize=3 2024-11-15T00:48:21,204 DEBUG [RS:0;fb2f6cb7846a:42925 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0, corePoolSize=3, maxPoolSize=3 2024-11-15T00:48:21,212 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:21,212 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:21,212 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:21,212 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:21,213 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:21,213 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,42925,1731631698709-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T00:48:21,235 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-15T00:48:21,237 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,42925,1731631698709-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:21,237 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:21,238 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.Replication(171): fb2f6cb7846a,42925,1731631698709 started 2024-11-15T00:48:21,256 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:21,257 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1482): Serving as fb2f6cb7846a,42925,1731631698709, RpcServer on fb2f6cb7846a/172.17.0.3:42925, sessionid=0x1013bf6c77b0001 2024-11-15T00:48:21,258 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-15T00:48:21,258 DEBUG [RS:0;fb2f6cb7846a:42925 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:21,258 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fb2f6cb7846a,42925,1731631698709' 2024-11-15T00:48:21,259 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-15T00:48:21,264 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-15T00:48:21,265 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-15T00:48:21,265 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-15T00:48:21,265 DEBUG [RS:0;fb2f6cb7846a:42925 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:21,265 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member 'fb2f6cb7846a,42925,1731631698709' 2024-11-15T00:48:21,265 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-15T00:48:21,266 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-15T00:48:21,266 DEBUG [RS:0;fb2f6cb7846a:42925 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-15T00:48:21,267 INFO [RS:0;fb2f6cb7846a:42925 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-15T00:48:21,267 INFO [RS:0;fb2f6cb7846a:42925 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-15T00:48:21,372 INFO [RS:0;fb2f6cb7846a:42925 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-15T00:48:21,376 INFO [RS:0;fb2f6cb7846a:42925 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fb2f6cb7846a%2C42925%2C1731631698709, suffix=, logDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/WALs/fb2f6cb7846a,42925,1731631698709, archiveDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/oldWALs, maxLogs=32 2024-11-15T00:48:21,384 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-15T00:48:21,385 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd 2024-11-15T00:48:21,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741832_1008 (size=32) 2024-11-15T00:48:21,394 DEBUG [RS:0;fb2f6cb7846a:42925 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/WALs/fb2f6cb7846a,42925,1731631698709/fb2f6cb7846a%2C42925%2C1731631698709.1731631701380, exclude list is [], retry=0 2024-11-15T00:48:21,400 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37847,DS-9a544359-9c18-4087-b458-d3b1e86c3abe,DISK] 2024-11-15T00:48:21,403 INFO [RS:0;fb2f6cb7846a:42925 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/WALs/fb2f6cb7846a,42925,1731631698709/fb2f6cb7846a%2C42925%2C1731631698709.1731631701380 2024-11-15T00:48:21,404 DEBUG [RS:0;fb2f6cb7846a:42925 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36083:36083)] 2024-11-15T00:48:21,796 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:48:21,798 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T00:48:21,801 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T00:48:21,801 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:21,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:48:21,802 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T00:48:21,805 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T00:48:21,805 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:21,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:48:21,807 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T00:48:21,810 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T00:48:21,810 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:21,812 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:48:21,813 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T00:48:21,816 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T00:48:21,816 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:21,817 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:48:21,818 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T00:48:21,820 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740 2024-11-15T00:48:21,821 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740 2024-11-15T00:48:21,830 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T00:48:21,830 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T00:48:21,831 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:48:21,836 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T00:48:21,845 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:48:21,847 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=58956856, jitterRate=-0.12147438526153564}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:48:21,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1731631701796Initializing all the Stores at 1731631701798 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631701798Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631701798Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631701798Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631701798Cleaning up temporary data from old regions at 1731631701830 (+32 ms)Region opened successfully at 1731631701850 (+20 ms) 2024-11-15T00:48:21,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T00:48:21,851 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T00:48:21,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T00:48:21,851 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T00:48:21,852 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T00:48:21,853 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T00:48:21,853 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731631701851Disabling compacts and flushes for region at 1731631701851Disabling writes for close at 1731631701851Writing region close event to WAL at 1731631701853 (+2 ms)Closed at 1731631701853 2024-11-15T00:48:21,857 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T00:48:21,858 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-15T00:48:21,865 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-15T00:48:21,874 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T00:48:21,878 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-15T00:48:22,031 DEBUG [fb2f6cb7846a:46397 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-15T00:48:22,045 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:22,056 INFO [PEWorker-3 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,42925,1731631698709, state=OPENING 2024-11-15T00:48:22,123 DEBUG [PEWorker-3 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-15T00:48:22,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:22,164 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:48:22,167 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:48:22,167 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:48:22,169 DEBUG [PEWorker-3 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-15T00:48:22,172 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:48:22,353 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-15T00:48:22,366 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:51113, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-15T00:48:22,381 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-15T00:48:22,382 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-15T00:48:22,382 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-15T00:48:22,386 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=fb2f6cb7846a%2C42925%2C1731631698709.meta, suffix=.meta, logDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/WALs/fb2f6cb7846a,42925,1731631698709, archiveDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/oldWALs, maxLogs=32 2024-11-15T00:48:22,407 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/WALs/fb2f6cb7846a,42925,1731631698709/fb2f6cb7846a%2C42925%2C1731631698709.meta.1731631702389.meta, exclude list is [], retry=0 2024-11-15T00:48:22,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:37847,DS-9a544359-9c18-4087-b458-d3b1e86c3abe,DISK] 2024-11-15T00:48:22,414 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/WALs/fb2f6cb7846a,42925,1731631698709/fb2f6cb7846a%2C42925%2C1731631698709.meta.1731631702389.meta 2024-11-15T00:48:22,415 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:36083:36083)] 2024-11-15T00:48:22,415 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:48:22,417 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-15T00:48:22,420 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-15T00:48:22,425 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-15T00:48:22,431 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-15T00:48:22,431 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:48:22,432 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-15T00:48:22,432 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-15T00:48:22,436 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-15T00:48:22,438 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-15T00:48:22,438 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:22,439 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:48:22,440 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-15T00:48:22,441 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-15T00:48:22,441 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:22,442 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:48:22,443 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-15T00:48:22,445 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-15T00:48:22,445 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:22,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:48:22,446 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-15T00:48:22,448 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-15T00:48:22,448 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:22,449 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-15T00:48:22,449 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-15T00:48:22,451 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740 2024-11-15T00:48:22,454 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740 2024-11-15T00:48:22,456 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-15T00:48:22,456 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-15T00:48:22,458 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:48:22,461 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-15T00:48:22,463 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62824672, jitterRate=-0.06383943557739258}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:48:22,463 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-15T00:48:22,464 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1731631702433Writing region info on filesystem at 1731631702433Initializing all the Stores at 1731631702435 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631702435Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631702436 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631702436Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1731631702436Cleaning up temporary data from old regions at 1731631702457 (+21 ms)Running coprocessor post-open hooks at 1731631702463 (+6 ms)Region opened successfully at 1731631702464 (+1 ms) 2024-11-15T00:48:22,471 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1731631702342 2024-11-15T00:48:22,481 DEBUG [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-15T00:48:22,481 INFO [RS_OPEN_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-15T00:48:22,483 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:22,485 INFO [PEWorker-5 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as fb2f6cb7846a,42925,1731631698709, state=OPEN 2024-11-15T00:48:22,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:48:22,588 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-15T00:48:22,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:48:22,588 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-15T00:48:22,589 DEBUG [PEWorker-5 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:22,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-15T00:48:22,602 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=fb2f6cb7846a,42925,1731631698709 in 418 msec 2024-11-15T00:48:22,611 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-15T00:48:22,612 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 739 msec 2024-11-15T00:48:22,615 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-15T00:48:22,615 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-15T00:48:22,647 DEBUG [PEWorker-1 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:22,649 DEBUG [PEWorker-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:22,673 DEBUG [PEWorker-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:22,677 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:54733, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:22,707 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.8340 sec 2024-11-15T00:48:22,707 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1731631702707, completionTime=-1 2024-11-15T00:48:22,710 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-15T00:48:22,711 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-15T00:48:22,787 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-15T00:48:22,787 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1731631762787 2024-11-15T00:48:22,787 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1731631822787 2024-11-15T00:48:22,787 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 76 msec 2024-11-15T00:48:22,790 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,46397,1731631697729-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:22,791 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,46397,1731631697729-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:22,791 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,46397,1731631697729-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:22,793 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-fb2f6cb7846a:46397, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:22,794 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:22,796 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:22,805 DEBUG [master/fb2f6cb7846a:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-15T00:48:22,831 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 3.861sec 2024-11-15T00:48:22,832 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-15T00:48:22,833 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-15T00:48:22,834 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-15T00:48:22,834 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-15T00:48:22,834 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-15T00:48:22,835 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,46397,1731631697729-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-15T00:48:22,835 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,46397,1731631697729-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-15T00:48:22,843 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-15T00:48:22,843 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-15T00:48:22,844 INFO [master/fb2f6cb7846a:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=fb2f6cb7846a,46397,1731631697729-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-15T00:48:22,937 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4799df63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:22,939 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-15T00:48:22,939 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-15T00:48:22,941 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:22,943 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:22,950 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:22,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:22,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d86f895, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:22,955 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:22,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:22,959 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:22,960 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:22,963 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f66000e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:22,963 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:22,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:22,973 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57504, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:22,989 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:23,000 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=838, ProcessCount=11, AvailableMemoryMB=3111 2024-11-15T00:48:23,007 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:48:23,012 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:23,015 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@65d48f8e 2024-11-15T00:48:23,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:48:23,019 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44088, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:48:23,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:48:23,026 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:48:23,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:48:23,036 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:48:23,038 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-15T00:48:23,039 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:23,041 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:48:23,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:48:23,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741835_1011 (size=959) 2024-11-15T00:48:23,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:48:23,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:48:23,457 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd 2024-11-15T00:48:23,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741836_1012 (size=53) 2024-11-15T00:48:23,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:48:23,882 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:48:23,883 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 5b99d5a4d9ee135a84ccb828f4037b20, disabling compactions & flushes 2024-11-15T00:48:23,883 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:23,883 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:23,883 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. after waiting 0 ms 2024-11-15T00:48:23,883 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:23,883 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:23,883 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 5b99d5a4d9ee135a84ccb828f4037b20: Waiting for close lock at 1731631703883Disabling compacts and flushes for region at 1731631703883Disabling writes for close at 1731631703883Writing region close event to WAL at 1731631703883Closed at 1731631703883 2024-11-15T00:48:23,886 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:48:23,893 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631703887"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631703887"}]},"ts":"1731631703887"} 2024-11-15T00:48:23,899 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:48:23,902 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:48:23,905 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631703902"}]},"ts":"1731631703902"} 2024-11-15T00:48:23,911 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:48:23,913 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b99d5a4d9ee135a84ccb828f4037b20, ASSIGN}] 2024-11-15T00:48:23,916 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b99d5a4d9ee135a84ccb828f4037b20, ASSIGN 2024-11-15T00:48:23,918 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b99d5a4d9ee135a84ccb828f4037b20, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,42925,1731631698709; forceNewPlan=false, retain=false 2024-11-15T00:48:24,070 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5b99d5a4d9ee135a84ccb828f4037b20, regionState=OPENING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:24,076 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b99d5a4d9ee135a84ccb828f4037b20, ASSIGN because future has completed 2024-11-15T00:48:24,077 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:48:24,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:48:24,239 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:24,240 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:48:24,240 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,240 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:48:24,241 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,241 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,243 INFO [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,246 INFO [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b99d5a4d9ee135a84ccb828f4037b20 columnFamilyName A 2024-11-15T00:48:24,246 DEBUG [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:24,248 INFO [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] regionserver.HStore(327): Store=5b99d5a4d9ee135a84ccb828f4037b20/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:24,248 INFO [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,251 INFO [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b99d5a4d9ee135a84ccb828f4037b20 columnFamilyName B 2024-11-15T00:48:24,252 DEBUG [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:24,253 INFO [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] regionserver.HStore(327): Store=5b99d5a4d9ee135a84ccb828f4037b20/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:24,253 INFO [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,256 INFO [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 5b99d5a4d9ee135a84ccb828f4037b20 columnFamilyName C 2024-11-15T00:48:24,257 DEBUG [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:24,258 INFO [StoreOpener-5b99d5a4d9ee135a84ccb828f4037b20-1 {}] regionserver.HStore(327): Store=5b99d5a4d9ee135a84ccb828f4037b20/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:24,258 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,261 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,264 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,267 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,267 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,269 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:48:24,273 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,285 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:48:24,286 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 5b99d5a4d9ee135a84ccb828f4037b20; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=66637962, jitterRate=-0.007016986608505249}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:48:24,287 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:24,288 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 5b99d5a4d9ee135a84ccb828f4037b20: Running coprocessor pre-open hook at 1731631704241Writing region info on filesystem at 1731631704241Initializing all the Stores at 1731631704243 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631704243Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631704243Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631704243Cleaning up temporary data from old regions at 1731631704267 (+24 ms)Running coprocessor post-open hooks at 1731631704287 (+20 ms)Region opened successfully at 1731631704288 (+1 ms) 2024-11-15T00:48:24,291 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., pid=6, masterSystemTime=1731631704233 2024-11-15T00:48:24,295 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:24,295 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:24,297 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=5b99d5a4d9ee135a84ccb828f4037b20, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:24,301 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:48:24,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-15T00:48:24,309 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 in 227 msec 2024-11-15T00:48:24,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-15T00:48:24,315 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b99d5a4d9ee135a84ccb828f4037b20, ASSIGN in 396 msec 2024-11-15T00:48:24,317 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:48:24,317 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631704317"}]},"ts":"1731631704317"} 2024-11-15T00:48:24,321 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:48:24,323 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:48:24,329 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2940 sec 2024-11-15T00:48:25,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-15T00:48:25,199 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:25,203 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b7cf82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,203 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:25,204 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:25,205 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:25,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:25,206 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29446fc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:25,207 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:25,208 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:25,209 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44116, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:25,210 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e077d21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,211 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:25,212 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:25,213 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:25,215 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57520, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:25,217 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:48:25,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:25,219 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4d4e458b 2024-11-15T00:48:25,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:48:25,224 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44126, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:48:25,228 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c22fa37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,228 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:25,228 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:25,230 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:25,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:25,230 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@787f40de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:25,231 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:25,233 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44146, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:25,234 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@740f65e4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,235 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7b17c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,235 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:25,235 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:25,237 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@58a08f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:25,239 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:25,241 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44160, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:25,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2544cc55, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,244 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6068bcec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,244 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:25,245 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:25,247 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:25,248 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79382fcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:25,250 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:25,252 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44176, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:25,254 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@596d06e1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,255 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fbf093, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,255 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:25,255 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:25,257 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e6ed1ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,257 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:25,258 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:25,260 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44198, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:25,261 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62dac934, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,263 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f1c2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,263 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:25,263 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:25,265 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3abd488a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,265 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:25,266 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:25,267 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:25,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@103656b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,270 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eea3ba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,270 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:25,270 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:25,271 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:25,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:25,272 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bbf9ca7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:25,273 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:25,274 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:25,275 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:25,276 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19664801, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,277 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6a39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,277 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:25,277 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:25,279 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:25,279 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:25,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44731cd9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:25,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:25,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:25,282 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:25,283 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@870009d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,285 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b7950b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,285 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:25,285 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:25,288 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:25,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d95629, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:25,290 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:25,291 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:25,293 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44260, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:25,294 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3837915e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,295 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b18ef51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,295 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:25,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:25,297 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40e873e9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:25,298 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:25,299 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:25,300 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:44270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:25,301 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21616163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:25,304 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:25,306 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:25,306 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:25,307 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:25,308 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:25,309 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:25,309 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57534, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:25,310 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57546, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:25,312 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:25,313 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:25,316 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57552, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:25,318 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:25,318 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:48:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:48:25,320 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:48:25,322 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:25,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:25,325 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:25,325 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:25,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57568, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:25,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57572, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:25,321 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:25,330 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:25,331 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:48:25,333 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57580, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:25,336 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:48:25,338 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:25,339 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:25,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57582, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:25,344 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:48:25,346 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:25,346 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:48:25,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-15T00:48:25,348 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:25,349 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:25,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:48:25,350 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:25,351 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:25,351 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:48:25,354 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:48:25,357 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:25,358 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:25,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:25,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:48:25,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:48:25,533 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:48:25,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:25,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:25,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:25,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:25,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:25,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:25,675 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/12ee2a5656d14d40ab286680006a3771 is 50, key is test_row_0/A:col10/1731631705414/Put/seqid=0 2024-11-15T00:48:25,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:48:25,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741837_1013 (size=12001) 2024-11-15T00:48:25,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/12ee2a5656d14d40ab286680006a3771 2024-11-15T00:48:25,732 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:48:25,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:25,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:25,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:25,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:25,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:25,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:25,785 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:25,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631765770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:25,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631765759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:25,804 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:25,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631765774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:25,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:25,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631765787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:25,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:25,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631765787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,851 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,853 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:25,912 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/537ecf1b00b74dae8d8dc88b5783604b is 50, key is test_row_0/B:col10/1731631705414/Put/seqid=0 2024-11-15T00:48:25,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:48:25,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:25,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:25,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:25,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:25,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:25,956 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741838_1014 (size=12001) 2024-11-15T00:48:25,965 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:25,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631765961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,966 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:25,966 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:25,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631765961, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:25,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,968 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:25,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:25,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631765960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631765962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:25,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:25,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631765962, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:25,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:25,977 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:25,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:48:26,079 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:48:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:26,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:26,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:26,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631766178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:26,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631766179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:26,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631766179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:26,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:26,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:26,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631766179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:26,188 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:26,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:26,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631766188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:26,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:48:26,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:26,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:26,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:26,252 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,365 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/537ecf1b00b74dae8d8dc88b5783604b 2024-11-15T00:48:26,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:48:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:26,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,448 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/2c4288d85b6444518b14e8c844ed8dd9 is 50, key is test_row_0/C:col10/1731631705414/Put/seqid=0 2024-11-15T00:48:26,494 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:26,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631766487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,495 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:26,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:48:26,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:26,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631766498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:26,500 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,501 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741839_1015 (size=12001) 2024-11-15T00:48:26,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:26,502 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=12 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/2c4288d85b6444518b14e8c844ed8dd9 2024-11-15T00:48:26,503 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:26,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631766498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,504 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:26,505 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:26,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631766499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,506 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:26,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:26,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631766507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:26,514 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:26,524 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/12ee2a5656d14d40ab286680006a3771 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/12ee2a5656d14d40ab286680006a3771 2024-11-15T00:48:26,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/12ee2a5656d14d40ab286680006a3771, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T00:48:26,559 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/537ecf1b00b74dae8d8dc88b5783604b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/537ecf1b00b74dae8d8dc88b5783604b 2024-11-15T00:48:26,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/537ecf1b00b74dae8d8dc88b5783604b, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T00:48:26,581 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/2c4288d85b6444518b14e8c844ed8dd9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2c4288d85b6444518b14e8c844ed8dd9 2024-11-15T00:48:26,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:48:26,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:26,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:26,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:26,582 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:26,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2c4288d85b6444518b14e8c844ed8dd9, entries=150, sequenceid=12, filesize=11.7 K 2024-11-15T00:48:26,645 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1177ms, sequenceid=12, compaction requested=false 2024-11-15T00:48:26,647 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T00:48:26,650 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:26,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-15T00:48:26,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:26,740 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:48:26,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/18f7532207124f05a772ac1ba99fd38b is 50, key is test_row_0/A:col10/1731631705775/Put/seqid=0 2024-11-15T00:48:26,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741840_1016 (size=12001) 2024-11-15T00:48:26,836 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/18f7532207124f05a772ac1ba99fd38b 2024-11-15T00:48:26,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/9eaa0ad88ea844c99a761b96ea715e6d is 50, key is test_row_0/B:col10/1731631705775/Put/seqid=0 2024-11-15T00:48:26,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741841_1017 (size=12001) 2024-11-15T00:48:26,962 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/9eaa0ad88ea844c99a761b96ea715e6d 2024-11-15T00:48:27,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/33e3b9a1fc6c4a5599c1b635344bfc32 is 50, key is test_row_0/C:col10/1731631705775/Put/seqid=0 2024-11-15T00:48:27,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:27,022 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:27,058 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741842_1018 (size=12001) 2024-11-15T00:48:27,060 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/33e3b9a1fc6c4a5599c1b635344bfc32 2024-11-15T00:48:27,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/18f7532207124f05a772ac1ba99fd38b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/18f7532207124f05a772ac1ba99fd38b 2024-11-15T00:48:27,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631767046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,090 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/18f7532207124f05a772ac1ba99fd38b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T00:48:27,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,091 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/9eaa0ad88ea844c99a761b96ea715e6d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9eaa0ad88ea844c99a761b96ea715e6d 2024-11-15T00:48:27,092 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631767055, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,095 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,101 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631767086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,103 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,113 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9eaa0ad88ea844c99a761b96ea715e6d, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T00:48:27,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/33e3b9a1fc6c4a5599c1b635344bfc32 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/33e3b9a1fc6c4a5599c1b635344bfc32 2024-11-15T00:48:27,129 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631767088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,131 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631767088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,132 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,140 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/33e3b9a1fc6c4a5599c1b635344bfc32, entries=150, sequenceid=37, filesize=11.7 K 2024-11-15T00:48:27,145 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=60.38 KB/61830 for 5b99d5a4d9ee135a84ccb828f4037b20 in 406ms, sequenceid=37, compaction requested=false 2024-11-15T00:48:27,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:27,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:27,145 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-15T00:48:27,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-15T00:48:27,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-15T00:48:27,155 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7910 sec 2024-11-15T00:48:27,160 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 1.8300 sec 2024-11-15T00:48:27,162 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-15T00:48:27,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:27,235 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:27,235 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T00:48:27,287 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/b57cd07500d84854bdd4d08ddca21ebd is 50, key is test_row_0/A:col10/1731631707083/Put/seqid=0 2024-11-15T00:48:27,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741843_1019 (size=14341) 2024-11-15T00:48:27,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/b57cd07500d84854bdd4d08ddca21ebd 2024-11-15T00:48:27,367 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/6d3adb63b6664d54aad03a1edb358ebd is 50, key is test_row_0/B:col10/1731631707083/Put/seqid=0 2024-11-15T00:48:27,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631767381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,409 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741844_1020 (size=12001) 2024-11-15T00:48:27,422 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/6d3adb63b6664d54aad03a1edb358ebd 2024-11-15T00:48:27,438 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631767402, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,439 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,441 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631767408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,442 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,443 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631767409, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,444 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,445 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,445 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631767411, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,446 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,468 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/c937bc877ea648949968fd9b212650bf is 50, key is test_row_0/C:col10/1731631707083/Put/seqid=0 2024-11-15T00:48:27,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-15T00:48:27,508 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:27,509 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-15T00:48:27,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T00:48:27,530 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631767520, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,534 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,535 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,556 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631767548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,558 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631767548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,559 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,560 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,574 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631767549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,575 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,576 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631767557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741845_1021 (size=12001) 2024-11-15T00:48:27,593 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/c937bc877ea648949968fd9b212650bf 2024-11-15T00:48:27,594 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:27,598 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:27,598 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:27,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T00:48:27,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/b57cd07500d84854bdd4d08ddca21ebd as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b57cd07500d84854bdd4d08ddca21ebd 2024-11-15T00:48:27,651 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b57cd07500d84854bdd4d08ddca21ebd, entries=200, sequenceid=50, filesize=14.0 K 2024-11-15T00:48:27,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/6d3adb63b6664d54aad03a1edb358ebd as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/6d3adb63b6664d54aad03a1edb358ebd 2024-11-15T00:48:27,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/6d3adb63b6664d54aad03a1edb358ebd, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T00:48:27,674 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/c937bc877ea648949968fd9b212650bf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c937bc877ea648949968fd9b212650bf 2024-11-15T00:48:27,693 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c937bc877ea648949968fd9b212650bf, entries=150, sequenceid=50, filesize=11.7 K 2024-11-15T00:48:27,705 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=147.60 KB/151140 for 5b99d5a4d9ee135a84ccb828f4037b20 in 470ms, sequenceid=50, compaction requested=true 2024-11-15T00:48:27,705 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:27,708 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:27,710 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:27,711 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:27,711 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:27,715 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:27,716 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:27,717 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/B is initiating minor compaction (all files) 2024-11-15T00:48:27,718 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/B in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:27,719 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/537ecf1b00b74dae8d8dc88b5783604b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9eaa0ad88ea844c99a761b96ea715e6d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/6d3adb63b6664d54aad03a1edb358ebd] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=35.2 K 2024-11-15T00:48:27,723 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 537ecf1b00b74dae8d8dc88b5783604b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731631705375 2024-11-15T00:48:27,723 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38343 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:27,723 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/A is initiating minor compaction (all files) 2024-11-15T00:48:27,723 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/A in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:27,724 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9eaa0ad88ea844c99a761b96ea715e6d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731631705749 2024-11-15T00:48:27,724 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/12ee2a5656d14d40ab286680006a3771, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/18f7532207124f05a772ac1ba99fd38b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b57cd07500d84854bdd4d08ddca21ebd] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=37.4 K 2024-11-15T00:48:27,725 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d3adb63b6664d54aad03a1edb358ebd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731631707040 2024-11-15T00:48:27,733 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 12ee2a5656d14d40ab286680006a3771, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731631705375 2024-11-15T00:48:27,737 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 18f7532207124f05a772ac1ba99fd38b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731631705749 2024-11-15T00:48:27,744 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b57cd07500d84854bdd4d08ddca21ebd, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731631707035 2024-11-15T00:48:27,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-15T00:48:27,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:27,765 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:48:27,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:27,779 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:27,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/8ec1d437e57847ebab95abdd7993369b is 50, key is test_row_0/A:col10/1731631707375/Put/seqid=0 2024-11-15T00:48:27,796 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#B#compaction#9 average throughput is 0.47 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:27,799 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/2ab47d7e90484e77986078d469533a4d is 50, key is test_row_0/B:col10/1731631707083/Put/seqid=0 2024-11-15T00:48:27,806 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#A#compaction#11 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:27,807 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/90f731425b384928a5f7aa7634d94d6e is 50, key is test_row_0/A:col10/1731631707083/Put/seqid=0 2024-11-15T00:48:27,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631767808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,830 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,836 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631767818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,837 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T00:48:27,844 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631767818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,846 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631767830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,853 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631767825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,857 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,858 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741846_1022 (size=12001) 2024-11-15T00:48:27,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741847_1023 (size=12104) 2024-11-15T00:48:27,911 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741848_1024 (size=12104) 2024-11-15T00:48:27,915 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/2ab47d7e90484e77986078d469533a4d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2ab47d7e90484e77986078d469533a4d 2024-11-15T00:48:27,946 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/90f731425b384928a5f7aa7634d94d6e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/90f731425b384928a5f7aa7634d94d6e 2024-11-15T00:48:27,948 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631767937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,949 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,950 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:48:27,951 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631767948, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,953 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,959 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/B of 5b99d5a4d9ee135a84ccb828f4037b20 into 2ab47d7e90484e77986078d469533a4d(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:27,959 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:27,965 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/A of 5b99d5a4d9ee135a84ccb828f4037b20 into 90f731425b384928a5f7aa7634d94d6e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:27,965 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:27,968 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/A, priority=13, startTime=1731631707707; duration=0sec 2024-11-15T00:48:27,969 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:27,969 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:27,969 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:27,973 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,973 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631767957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,974 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/C is initiating minor compaction (all files) 2024-11-15T00:48:27,974 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/C in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:27,974 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2c4288d85b6444518b14e8c844ed8dd9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/33e3b9a1fc6c4a5599c1b635344bfc32, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c937bc877ea648949968fd9b212650bf] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=35.2 K 2024-11-15T00:48:27,974 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631767968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,975 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2c4288d85b6444518b14e8c844ed8dd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=12, earliestPutTs=1731631705375 2024-11-15T00:48:27,975 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:27,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631767971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,976 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 33e3b9a1fc6c4a5599c1b635344bfc32, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1731631705749 2024-11-15T00:48:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:27,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:27,977 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c937bc877ea648949968fd9b212650bf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731631707040 2024-11-15T00:48:27,977 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/B, priority=13, startTime=1731631707711; duration=0sec 2024-11-15T00:48:27,978 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:27,979 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:B 2024-11-15T00:48:28,027 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#C#compaction#12 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:28,028 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/bdf81e80e8e047d5b51295dfb6f7a963 is 50, key is test_row_0/C:col10/1731631707083/Put/seqid=0 2024-11-15T00:48:28,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741849_1025 (size=12104) 2024-11-15T00:48:28,082 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/bdf81e80e8e047d5b51295dfb6f7a963 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/bdf81e80e8e047d5b51295dfb6f7a963 2024-11-15T00:48:28,101 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/C of 5b99d5a4d9ee135a84ccb828f4037b20 into bdf81e80e8e047d5b51295dfb6f7a963(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:28,102 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:28,102 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/C, priority=13, startTime=1731631707716; duration=0sec 2024-11-15T00:48:28,102 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:28,102 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:C 2024-11-15T00:48:28,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T00:48:28,170 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:28,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631768164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,171 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:28,172 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:28,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631768165, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:28,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,173 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:28,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:28,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631768188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:28,193 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,194 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:28,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:28,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631768188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,195 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:28,195 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:28,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631768189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,196 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:28,197 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-15T00:48:28,197 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-15T00:48:28,200 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T00:48:28,200 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-15T00:48:28,201 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-15T00:48:28,201 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-15T00:48:28,203 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T00:48:28,203 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T00:48:28,277 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/8ec1d437e57847ebab95abdd7993369b 2024-11-15T00:48:28,299 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/883249b33c074f50b8ee08cf29af2a2c is 50, key is test_row_0/B:col10/1731631707375/Put/seqid=0 2024-11-15T00:48:28,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741850_1026 (size=12001) 2024-11-15T00:48:28,343 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/883249b33c074f50b8ee08cf29af2a2c 2024-11-15T00:48:28,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/2e2060f659e34d2193e7644aac39269f is 50, key is test_row_0/C:col10/1731631707375/Put/seqid=0 2024-11-15T00:48:28,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741851_1027 (size=12001) 2024-11-15T00:48:28,424 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=75 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/2e2060f659e34d2193e7644aac39269f 2024-11-15T00:48:28,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/8ec1d437e57847ebab95abdd7993369b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8ec1d437e57847ebab95abdd7993369b 2024-11-15T00:48:28,471 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8ec1d437e57847ebab95abdd7993369b, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T00:48:28,474 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/883249b33c074f50b8ee08cf29af2a2c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/883249b33c074f50b8ee08cf29af2a2c 2024-11-15T00:48:28,487 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:28,487 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:28,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631768481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:28,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631768481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,488 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:28,491 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/883249b33c074f50b8ee08cf29af2a2c, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T00:48:28,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/2e2060f659e34d2193e7644aac39269f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2e2060f659e34d2193e7644aac39269f 2024-11-15T00:48:28,501 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:28,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631768497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:28,502 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,503 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:28,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631768510, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:28,512 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:28,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631768509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,513 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:28,514 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:28,517 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2e2060f659e34d2193e7644aac39269f, entries=150, sequenceid=75, filesize=11.7 K 2024-11-15T00:48:28,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 5b99d5a4d9ee135a84ccb828f4037b20 in 754ms, sequenceid=75, compaction requested=false 2024-11-15T00:48:28,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:28,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:28,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-15T00:48:28,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-15T00:48:28,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-15T00:48:28,529 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 925 msec 2024-11-15T00:48:28,536 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.0220 sec 2024-11-15T00:48:28,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-15T00:48:28,658 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:28,658 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:28,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-15T00:48:28,663 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:28,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:48:28,670 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:28,670 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:28,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:48:28,827 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-15T00:48:28,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:28,831 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:48:28,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/d2446edcbe68451f972cadaf31191c66 is 50, key is test_row_0/A:col10/1731631707819/Put/seqid=0 2024-11-15T00:48:28,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741852_1028 (size=12001) 2024-11-15T00:48:28,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:48:29,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:29,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631769127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,138 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631769125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,141 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,142 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631769128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,143 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,143 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631769132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631769136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,144 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,148 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,152 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,255 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631769249, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,256 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,256 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631769250, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,258 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,267 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631769257, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631769258, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,268 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631769261, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,268 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,269 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/d2446edcbe68451f972cadaf31191c66 2024-11-15T00:48:29,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:48:29,323 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/f899d544990f43a3acd3f8ebcc5dee4a is 50, key is test_row_0/B:col10/1731631707819/Put/seqid=0 2024-11-15T00:48:29,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741853_1029 (size=12001) 2024-11-15T00:48:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,469 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631769467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631769467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,470 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,472 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631769477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,483 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631769477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,484 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631769477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,486 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,793 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631769781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,795 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/f899d544990f43a3acd3f8ebcc5dee4a 2024-11-15T00:48:29,787 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631769781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,796 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631769791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631769792, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,805 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:48:29,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:29,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631769801, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:29,814 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:29,839 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/60103e43a2eb411ca7eaf59b4df652eb is 50, key is test_row_0/C:col10/1731631707819/Put/seqid=0 2024-11-15T00:48:29,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741854_1030 (size=12001) 2024-11-15T00:48:29,881 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/60103e43a2eb411ca7eaf59b4df652eb 2024-11-15T00:48:29,907 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/d2446edcbe68451f972cadaf31191c66 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d2446edcbe68451f972cadaf31191c66 2024-11-15T00:48:29,931 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d2446edcbe68451f972cadaf31191c66, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T00:48:29,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/f899d544990f43a3acd3f8ebcc5dee4a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/f899d544990f43a3acd3f8ebcc5dee4a 2024-11-15T00:48:29,966 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/f899d544990f43a3acd3f8ebcc5dee4a, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T00:48:29,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/60103e43a2eb411ca7eaf59b4df652eb as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/60103e43a2eb411ca7eaf59b4df652eb 2024-11-15T00:48:29,996 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/60103e43a2eb411ca7eaf59b4df652eb, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T00:48:30,004 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1167ms, sequenceid=89, compaction requested=true 2024-11-15T00:48:30,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:30,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:30,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-15T00:48:30,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-15T00:48:30,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-15T00:48:30,013 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3380 sec 2024-11-15T00:48:30,026 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3620 sec 2024-11-15T00:48:30,315 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:30,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T00:48:30,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:30,340 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/09eb8a4a064c486baba88daf0cb86bbf is 50, key is test_row_0/A:col10/1731631709114/Put/seqid=0 2024-11-15T00:48:30,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631770348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,369 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631770352, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,370 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631770353, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,372 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,379 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631770366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,380 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741855_1031 (size=14341) 2024-11-15T00:48:30,389 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631770373, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,396 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,484 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631770477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,486 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631770477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,487 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631770478, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,489 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631770487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,507 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,508 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,529 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631770509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,531 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,715 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631770709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,716 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,717 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631770711, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,718 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631770709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631770717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,725 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,740 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:30,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631770737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:30,747 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:30,785 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/09eb8a4a064c486baba88daf0cb86bbf 2024-11-15T00:48:30,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-15T00:48:30,821 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:30,822 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:30,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-15T00:48:30,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:48:30,831 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:30,833 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:30,833 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:30,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/71643fb6f18e43bd968eb7f070d08112 is 50, key is test_row_0/B:col10/1731631709114/Put/seqid=0 2024-11-15T00:48:30,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741856_1032 (size=12001) 2024-11-15T00:48:30,886 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/71643fb6f18e43bd968eb7f070d08112 2024-11-15T00:48:30,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:48:30,979 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/adb8fb5817db47b289ba04a619248780 is 50, key is test_row_0/C:col10/1731631709114/Put/seqid=0 2024-11-15T00:48:30,993 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:48:30,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:30,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:30,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741857_1033 (size=12001) 2024-11-15T00:48:31,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:31,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631771029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:31,041 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631771030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:31,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631771030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,044 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:31,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:31,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631771040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,055 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,056 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:31,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:31,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631771057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,068 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:31,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:48:31,164 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:48:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:31,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:31,174 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:31,174 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,175 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,332 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:48:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:31,332 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:31,332 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,333 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/adb8fb5817db47b289ba04a619248780 2024-11-15T00:48:31,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:48:31,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:48:31,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:31,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:31,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:31,490 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,490 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/09eb8a4a064c486baba88daf0cb86bbf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/09eb8a4a064c486baba88daf0cb86bbf 2024-11-15T00:48:31,490 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:31,541 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/09eb8a4a064c486baba88daf0cb86bbf, entries=200, sequenceid=115, filesize=14.0 K 2024-11-15T00:48:31,557 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/71643fb6f18e43bd968eb7f070d08112 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/71643fb6f18e43bd968eb7f070d08112 2024-11-15T00:48:31,560 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:31,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631771553, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:31,566 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:31,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631771561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,567 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,568 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:31,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:31,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631771580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:31,591 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/71643fb6f18e43bd968eb7f070d08112, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T00:48:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:31,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:31,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631771580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:31,594 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:31,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631771584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:31,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:31,596 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:31,596 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:31,599 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/adb8fb5817db47b289ba04a619248780 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adb8fb5817db47b289ba04a619248780 2024-11-15T00:48:31,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adb8fb5817db47b289ba04a619248780, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T00:48:31,635 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1319ms, sequenceid=115, compaction requested=true 2024-11-15T00:48:31,635 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:31,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:31,641 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:48:31,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:31,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:31,641 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:48:31,641 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:31,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:31,642 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:31,645 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50447 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:48:31,645 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/A is initiating minor compaction (all files) 2024-11-15T00:48:31,645 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/A in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:31,645 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/90f731425b384928a5f7aa7634d94d6e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8ec1d437e57847ebab95abdd7993369b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d2446edcbe68451f972cadaf31191c66, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/09eb8a4a064c486baba88daf0cb86bbf] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=49.3 K 2024-11-15T00:48:31,646 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 90f731425b384928a5f7aa7634d94d6e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731631707040 2024-11-15T00:48:31,647 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8ec1d437e57847ebab95abdd7993369b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731631707375 2024-11-15T00:48:31,648 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d2446edcbe68451f972cadaf31191c66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731631707807 2024-11-15T00:48:31,649 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-15T00:48:31,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:31,650 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:48:31,653 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 09eb8a4a064c486baba88daf0cb86bbf, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731631709114 2024-11-15T00:48:31,660 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:48:31,660 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/B is initiating minor compaction (all files) 2024-11-15T00:48:31,660 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/B in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:31,660 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2ab47d7e90484e77986078d469533a4d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/883249b33c074f50b8ee08cf29af2a2c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/f899d544990f43a3acd3f8ebcc5dee4a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/71643fb6f18e43bd968eb7f070d08112] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=47.0 K 2024-11-15T00:48:31,668 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ab47d7e90484e77986078d469533a4d, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731631707040 2024-11-15T00:48:31,676 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 883249b33c074f50b8ee08cf29af2a2c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731631707375 2024-11-15T00:48:31,685 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f899d544990f43a3acd3f8ebcc5dee4a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731631707807 2024-11-15T00:48:31,691 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 71643fb6f18e43bd968eb7f070d08112, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731631709114 2024-11-15T00:48:31,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/018070c79c484ec2972418d782077bd3 is 50, key is test_row_0/A:col10/1731631710339/Put/seqid=0 2024-11-15T00:48:31,731 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#A#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:31,732 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/36ade1a1f41a40258aaf2efb5b756d83 is 50, key is test_row_0/A:col10/1731631709114/Put/seqid=0 2024-11-15T00:48:31,736 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#B#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:31,737 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/e349a758f5104b90a03077da2ef0bbd3 is 50, key is test_row_0/B:col10/1731631709114/Put/seqid=0 2024-11-15T00:48:31,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741858_1034 (size=12001) 2024-11-15T00:48:31,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741859_1035 (size=12241) 2024-11-15T00:48:31,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741860_1036 (size=12241) 2024-11-15T00:48:31,830 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/36ade1a1f41a40258aaf2efb5b756d83 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/36ade1a1f41a40258aaf2efb5b756d83 2024-11-15T00:48:31,854 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/A of 5b99d5a4d9ee135a84ccb828f4037b20 into 36ade1a1f41a40258aaf2efb5b756d83(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:31,854 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:31,854 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/A, priority=12, startTime=1731631711641; duration=0sec 2024-11-15T00:48:31,855 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:31,855 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:31,855 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:48:31,859 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:48:31,859 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/C is initiating minor compaction (all files) 2024-11-15T00:48:31,860 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/C in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:31,860 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/bdf81e80e8e047d5b51295dfb6f7a963, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2e2060f659e34d2193e7644aac39269f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/60103e43a2eb411ca7eaf59b4df652eb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adb8fb5817db47b289ba04a619248780] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=47.0 K 2024-11-15T00:48:31,861 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting bdf81e80e8e047d5b51295dfb6f7a963, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1731631707040 2024-11-15T00:48:31,861 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2e2060f659e34d2193e7644aac39269f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=75, earliestPutTs=1731631707375 2024-11-15T00:48:31,862 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 60103e43a2eb411ca7eaf59b4df652eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731631707807 2024-11-15T00:48:31,863 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting adb8fb5817db47b289ba04a619248780, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731631709114 2024-11-15T00:48:31,915 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#C#compaction#24 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:31,915 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/adf5c9116e744a03b78023424e917091 is 50, key is test_row_0/C:col10/1731631709114/Put/seqid=0 2024-11-15T00:48:31,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:48:31,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741861_1037 (size=12241) 2024-11-15T00:48:32,019 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/adf5c9116e744a03b78023424e917091 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adf5c9116e744a03b78023424e917091 2024-11-15T00:48:32,045 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/C of 5b99d5a4d9ee135a84ccb828f4037b20 into adf5c9116e744a03b78023424e917091(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:32,045 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:32,045 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/C, priority=12, startTime=1731631711642; duration=0sec 2024-11-15T00:48:32,046 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:32,046 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:C 2024-11-15T00:48:32,141 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/018070c79c484ec2972418d782077bd3 2024-11-15T00:48:32,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/9a5d06ec68834c57af4f2e0526e9277b is 50, key is test_row_0/B:col10/1731631710339/Put/seqid=0 2024-11-15T00:48:32,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741862_1038 (size=12001) 2024-11-15T00:48:32,231 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/e349a758f5104b90a03077da2ef0bbd3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/e349a758f5104b90a03077da2ef0bbd3 2024-11-15T00:48:32,247 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/B of 5b99d5a4d9ee135a84ccb828f4037b20 into e349a758f5104b90a03077da2ef0bbd3(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:32,247 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:32,247 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/B, priority=12, startTime=1731631711641; duration=0sec 2024-11-15T00:48:32,248 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:32,248 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:B 2024-11-15T00:48:32,595 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:32,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:32,596 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/9a5d06ec68834c57af4f2e0526e9277b 2024-11-15T00:48:32,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/c97cb1d8672546ef9dbdeb2209b72f40 is 50, key is test_row_0/C:col10/1731631710339/Put/seqid=0 2024-11-15T00:48:32,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741863_1039 (size=12001) 2024-11-15T00:48:32,716 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/c97cb1d8672546ef9dbdeb2209b72f40 2024-11-15T00:48:32,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631772703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,736 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631772701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631772703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,737 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:32,737 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631772735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631772735, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:32,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,740 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:32,749 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/018070c79c484ec2972418d782077bd3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/018070c79c484ec2972418d782077bd3 2024-11-15T00:48:32,765 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/018070c79c484ec2972418d782077bd3, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T00:48:32,767 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/9a5d06ec68834c57af4f2e0526e9277b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9a5d06ec68834c57af4f2e0526e9277b 2024-11-15T00:48:32,786 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9a5d06ec68834c57af4f2e0526e9277b, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T00:48:32,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/c97cb1d8672546ef9dbdeb2209b72f40 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c97cb1d8672546ef9dbdeb2209b72f40 2024-11-15T00:48:32,813 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c97cb1d8672546ef9dbdeb2209b72f40, entries=150, sequenceid=126, filesize=11.7 K 2024-11-15T00:48:32,815 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1166ms, sequenceid=126, compaction requested=false 2024-11-15T00:48:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:32,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:32,816 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-15T00:48:32,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-15T00:48:32,822 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-15T00:48:32,823 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9860 sec 2024-11-15T00:48:32,829 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 2.0030 sec 2024-11-15T00:48:32,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:32,854 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:32,855 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T00:48:32,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,881 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/6f7970cf64444570a192a6abdf6d3f3f is 50, key is test_row_0/A:col10/1731631712704/Put/seqid=0 2024-11-15T00:48:32,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631772869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:32,883 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631772869, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,887 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:32,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631772875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,889 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,889 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,890 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631772878, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631772882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,891 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:32,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:32,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741864_1040 (size=12151) 2024-11-15T00:48:32,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-15T00:48:32,986 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:32,987 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:32,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-15T00:48:32,991 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:32,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:48:32,993 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:32,994 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:32,995 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:32,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631772987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:32,996 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,003 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631772997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,004 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631772998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,005 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,005 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631772998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,006 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,006 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631772998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,010 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,011 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:48:33,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T00:48:33,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:33,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:33,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:33,165 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:33,165 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:33,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:33,220 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631773217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,221 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,225 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631773225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,226 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631773225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,227 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631773225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,228 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,228 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631773225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,232 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:48:33,312 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/6f7970cf64444570a192a6abdf6d3f3f 2024-11-15T00:48:33,341 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T00:48:33,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:33,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:33,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:33,343 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:33,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:33,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:33,355 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/620a996a50524b7e92ec2c1362ba74b0 is 50, key is test_row_0/B:col10/1731631712704/Put/seqid=0 2024-11-15T00:48:33,394 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741865_1041 (size=12151) 2024-11-15T00:48:33,411 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/620a996a50524b7e92ec2c1362ba74b0 2024-11-15T00:48:33,447 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/4e90f7d9b1644563b938d83c8ff6781d is 50, key is test_row_0/C:col10/1731631712704/Put/seqid=0 2024-11-15T00:48:33,519 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T00:48:33,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:33,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741866_1042 (size=12151) 2024-11-15T00:48:33,519 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:33,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:33,520 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] handler.RSProcedureHandler(58): pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:33,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=16 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:33,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=16 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:33,532 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/4e90f7d9b1644563b938d83c8ff6781d 2024-11-15T00:48:33,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631773529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,542 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631773538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,542 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631773538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,543 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631773538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,544 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:33,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631773538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:33,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:33,545 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:33,569 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/6f7970cf64444570a192a6abdf6d3f3f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6f7970cf64444570a192a6abdf6d3f3f 2024-11-15T00:48:33,605 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6f7970cf64444570a192a6abdf6d3f3f, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T00:48:33,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/620a996a50524b7e92ec2c1362ba74b0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/620a996a50524b7e92ec2c1362ba74b0 2024-11-15T00:48:33,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:48:33,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/620a996a50524b7e92ec2c1362ba74b0, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T00:48:33,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/4e90f7d9b1644563b938d83c8ff6781d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4e90f7d9b1644563b938d83c8ff6781d 2024-11-15T00:48:33,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4e90f7d9b1644563b938d83c8ff6781d, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T00:48:33,644 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 5b99d5a4d9ee135a84ccb828f4037b20 in 789ms, sequenceid=155, compaction requested=true 2024-11-15T00:48:33,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:33,644 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:33,646 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:33,647 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/A is initiating minor compaction (all files) 2024-11-15T00:48:33,647 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/A in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:33,647 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/36ade1a1f41a40258aaf2efb5b756d83, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/018070c79c484ec2972418d782077bd3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6f7970cf64444570a192a6abdf6d3f3f] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=35.5 K 2024-11-15T00:48:33,648 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36ade1a1f41a40258aaf2efb5b756d83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731631709114 2024-11-15T00:48:33,649 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 018070c79c484ec2972418d782077bd3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731631710339 2024-11-15T00:48:33,650 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f7970cf64444570a192a6abdf6d3f3f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631712699 2024-11-15T00:48:33,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:33,676 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:33,676 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:33,681 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:33,681 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/B is initiating minor compaction (all files) 2024-11-15T00:48:33,682 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/B in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:33,682 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/e349a758f5104b90a03077da2ef0bbd3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9a5d06ec68834c57af4f2e0526e9277b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/620a996a50524b7e92ec2c1362ba74b0] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=35.5 K 2024-11-15T00:48:33,685 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-15T00:48:33,686 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e349a758f5104b90a03077da2ef0bbd3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731631709114 2024-11-15T00:48:33,686 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:33,686 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T00:48:33,687 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9a5d06ec68834c57af4f2e0526e9277b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731631710339 2024-11-15T00:48:33,688 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 620a996a50524b7e92ec2c1362ba74b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631712699 2024-11-15T00:48:33,708 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#A#compaction#30 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:33,709 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/9af89a258442452687475e35e96c0743 is 50, key is test_row_0/A:col10/1731631712704/Put/seqid=0 2024-11-15T00:48:33,709 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:33,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/b967f501065a4ea08636e0390fc93a0b is 50, key is test_row_0/A:col10/1731631712875/Put/seqid=0 2024-11-15T00:48:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:33,712 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:33,716 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#B#compaction#32 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:33,716 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/2a6c1ec6d3104b6ebd4025c3086ac56c is 50, key is test_row_0/B:col10/1731631712704/Put/seqid=0 2024-11-15T00:48:33,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741867_1043 (size=12493) 2024-11-15T00:48:33,797 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/9af89a258442452687475e35e96c0743 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/9af89a258442452687475e35e96c0743 2024-11-15T00:48:33,821 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/A of 5b99d5a4d9ee135a84ccb828f4037b20 into 9af89a258442452687475e35e96c0743(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:33,821 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:33,821 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/A, priority=13, startTime=1731631713644; duration=0sec 2024-11-15T00:48:33,821 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:33,821 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:33,821 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:33,823 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:33,823 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/C is initiating minor compaction (all files) 2024-11-15T00:48:33,823 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/C in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:33,824 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adf5c9116e744a03b78023424e917091, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c97cb1d8672546ef9dbdeb2209b72f40, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4e90f7d9b1644563b938d83c8ff6781d] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=35.5 K 2024-11-15T00:48:33,824 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting adf5c9116e744a03b78023424e917091, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731631709114 2024-11-15T00:48:33,825 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741868_1044 (size=12151) 2024-11-15T00:48:33,826 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c97cb1d8672546ef9dbdeb2209b72f40, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1731631710339 2024-11-15T00:48:33,828 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4e90f7d9b1644563b938d83c8ff6781d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631712699 2024-11-15T00:48:33,828 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/b967f501065a4ea08636e0390fc93a0b 2024-11-15T00:48:33,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741869_1045 (size=12493) 2024-11-15T00:48:33,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/fdc88f51de814ee2af46bbd046c144c4 is 50, key is test_row_0/B:col10/1731631712875/Put/seqid=0 2024-11-15T00:48:33,871 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#C#compaction#34 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:33,872 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/66a249b74bbd42d3883d132171e00f31 is 50, key is test_row_0/C:col10/1731631712704/Put/seqid=0 2024-11-15T00:48:33,902 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/2a6c1ec6d3104b6ebd4025c3086ac56c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2a6c1ec6d3104b6ebd4025c3086ac56c 2024-11-15T00:48:33,914 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/B of 5b99d5a4d9ee135a84ccb828f4037b20 into 2a6c1ec6d3104b6ebd4025c3086ac56c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:33,914 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:33,914 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/B, priority=13, startTime=1731631713676; duration=0sec 2024-11-15T00:48:33,914 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:33,914 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:B 2024-11-15T00:48:33,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741870_1046 (size=12151) 2024-11-15T00:48:33,931 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741871_1047 (size=12493) 2024-11-15T00:48:33,959 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/66a249b74bbd42d3883d132171e00f31 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/66a249b74bbd42d3883d132171e00f31 2024-11-15T00:48:33,979 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/C of 5b99d5a4d9ee135a84ccb828f4037b20 into 66a249b74bbd42d3883d132171e00f31(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:33,979 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:33,979 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/C, priority=13, startTime=1731631713712; duration=0sec 2024-11-15T00:48:33,980 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:33,980 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:C 2024-11-15T00:48:34,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:34,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:34,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:48:34,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631774168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,189 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,189 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631774169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,190 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631774164, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631774184, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,197 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631774185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,304 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631774298, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,305 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631774308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,313 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631774308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,313 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631774308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,317 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,318 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,316 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631774308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,321 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,332 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/fdc88f51de814ee2af46bbd046c144c4 2024-11-15T00:48:34,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/5aaf5c32b340489f884c294f3888b094 is 50, key is test_row_0/C:col10/1731631712875/Put/seqid=0 2024-11-15T00:48:34,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741872_1048 (size=12151) 2024-11-15T00:48:34,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631774519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,530 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631774527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,531 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,532 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,532 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631774529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,533 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631774529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,534 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,537 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631774537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,549 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,817 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/5aaf5c32b340489f884c294f3888b094 2024-11-15T00:48:34,851 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631774844, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,852 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,855 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631774850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,856 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631774852, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631774853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,864 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:34,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631774857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:34,870 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:34,873 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/b967f501065a4ea08636e0390fc93a0b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b967f501065a4ea08636e0390fc93a0b 2024-11-15T00:48:34,895 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b967f501065a4ea08636e0390fc93a0b, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T00:48:34,902 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/fdc88f51de814ee2af46bbd046c144c4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/fdc88f51de814ee2af46bbd046c144c4 2024-11-15T00:48:34,938 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/fdc88f51de814ee2af46bbd046c144c4, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T00:48:34,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/5aaf5c32b340489f884c294f3888b094 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5aaf5c32b340489f884c294f3888b094 2024-11-15T00:48:34,972 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5aaf5c32b340489f884c294f3888b094, entries=150, sequenceid=165, filesize=11.9 K 2024-11-15T00:48:34,975 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=161.02 KB/164880 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1289ms, sequenceid=165, compaction requested=false 2024-11-15T00:48:34,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:34,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:34,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-15T00:48:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-15T00:48:34,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-15T00:48:34,984 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9850 sec 2024-11-15T00:48:34,995 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 2.0020 sec 2024-11-15T00:48:35,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-15T00:48:35,158 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:35,159 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:35,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-15T00:48:35,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:48:35,184 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:35,188 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:35,189 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:35,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:48:35,364 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-15T00:48:35,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:35,367 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T00:48:35,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:35,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:35,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/32208904cbc745f8860dab4737b6b0ec is 50, key is test_row_0/A:col10/1731631714161/Put/seqid=0 2024-11-15T00:48:35,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741873_1049 (size=12151) 2024-11-15T00:48:35,491 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631775476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,492 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:48:35,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631775472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,509 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631775492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,510 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631775490, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,514 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631775477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,518 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,519 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,597 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631775597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,605 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,606 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631775621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,627 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,627 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631775621, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,632 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631775624, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,633 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631775629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,641 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:48:35,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631775819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,832 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631775839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631775839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,848 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/32208904cbc745f8860dab4737b6b0ec 2024-11-15T00:48:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,849 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631775840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,850 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:35,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631775848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:35,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:35,864 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:35,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/966c910f54324219a8fbc87ec0ba0d75 is 50, key is test_row_0/B:col10/1731631714161/Put/seqid=0 2024-11-15T00:48:35,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741874_1050 (size=12151) 2024-11-15T00:48:36,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:36,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631776137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,139 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:36,161 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:36,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631776156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,162 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:36,162 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:36,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631776157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:36,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631776157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,163 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:36,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631776180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:36,188 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,189 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:36,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:48:36,386 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/966c910f54324219a8fbc87ec0ba0d75 2024-11-15T00:48:36,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/7e5e56adffd44e87a842d7d35d78cb6c is 50, key is test_row_0/C:col10/1731631714161/Put/seqid=0 2024-11-15T00:48:36,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741875_1051 (size=12151) 2024-11-15T00:48:36,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:36,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631776646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:36,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:36,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631776667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,671 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,672 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:36,673 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:36,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631776672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:36,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631776672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,674 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:36,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:36,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631776699, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:36,701 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:36,865 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/7e5e56adffd44e87a842d7d35d78cb6c 2024-11-15T00:48:36,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/32208904cbc745f8860dab4737b6b0ec as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/32208904cbc745f8860dab4737b6b0ec 2024-11-15T00:48:36,889 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/32208904cbc745f8860dab4737b6b0ec, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T00:48:36,892 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/966c910f54324219a8fbc87ec0ba0d75 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/966c910f54324219a8fbc87ec0ba0d75 2024-11-15T00:48:36,908 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/966c910f54324219a8fbc87ec0ba0d75, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T00:48:36,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/7e5e56adffd44e87a842d7d35d78cb6c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/7e5e56adffd44e87a842d7d35d78cb6c 2024-11-15T00:48:36,923 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/7e5e56adffd44e87a842d7d35d78cb6c, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T00:48:36,925 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1559ms, sequenceid=195, compaction requested=true 2024-11-15T00:48:36,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:36,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:36,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-15T00:48:36,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-15T00:48:36,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-15T00:48:36,936 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7410 sec 2024-11-15T00:48:36,939 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.7780 sec 2024-11-15T00:48:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-15T00:48:37,338 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:37,339 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:37,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-15T00:48:37,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:48:37,344 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:37,345 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:37,345 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:37,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:48:37,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-15T00:48:37,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:37,501 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T00:48:37,522 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/1b20fccf80474ff6be84695122d16556 is 50, key is test_row_0/A:col10/1731631715397/Put/seqid=0 2024-11-15T00:48:37,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741876_1052 (size=12151) 2024-11-15T00:48:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:48:37,681 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:37,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:37,783 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631777778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,785 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:37,800 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631777784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:37,800 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:37,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631777782, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,801 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:37,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631777784, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,801 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,805 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631777824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,829 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:37,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:37,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631777901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:37,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:37,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631777907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,910 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:37,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:37,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631777908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:37,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:37,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:37,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631777911, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,916 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,917 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:37,947 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:37,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631777940, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:37,955 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:37,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:48:37,972 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/1b20fccf80474ff6be84695122d16556 2024-11-15T00:48:38,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/0a576c2f8d4749a7b1415653004341ac is 50, key is test_row_0/B:col10/1731631715397/Put/seqid=0 2024-11-15T00:48:38,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741877_1053 (size=12151) 2024-11-15T00:48:38,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631778117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631778120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631778120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,129 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631778128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,131 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,132 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,172 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631778168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,185 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,446 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631778438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,448 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631778438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,449 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631778438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631778438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,455 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,457 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,475 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/0a576c2f8d4749a7b1415653004341ac 2024-11-15T00:48:38,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:48:38,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631778488, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,496 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/caf12d168bce464281c8f54b9f358a16 is 50, key is test_row_0/C:col10/1731631715397/Put/seqid=0 2024-11-15T00:48:38,563 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741878_1054 (size=12151) 2024-11-15T00:48:38,962 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631778957, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,968 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631778967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,969 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631778967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,970 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:38,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631778968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:38,971 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:38,972 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/caf12d168bce464281c8f54b9f358a16 2024-11-15T00:48:39,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/1b20fccf80474ff6be84695122d16556 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/1b20fccf80474ff6be84695122d16556 2024-11-15T00:48:39,009 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:39,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631779007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:39,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:39,018 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/1b20fccf80474ff6be84695122d16556, entries=150, sequenceid=204, filesize=11.9 K 2024-11-15T00:48:39,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/0a576c2f8d4749a7b1415653004341ac as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/0a576c2f8d4749a7b1415653004341ac 2024-11-15T00:48:39,056 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/0a576c2f8d4749a7b1415653004341ac, entries=150, sequenceid=204, filesize=11.9 K 2024-11-15T00:48:39,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/caf12d168bce464281c8f54b9f358a16 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/caf12d168bce464281c8f54b9f358a16 2024-11-15T00:48:39,069 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/caf12d168bce464281c8f54b9f358a16, entries=150, sequenceid=204, filesize=11.9 K 2024-11-15T00:48:39,070 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=167.72 KB/171750 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1570ms, sequenceid=204, compaction requested=true 2024-11-15T00:48:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:39,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-15T00:48:39,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-15T00:48:39,082 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-15T00:48:39,083 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7310 sec 2024-11-15T00:48:39,099 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.7550 sec 2024-11-15T00:48:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-15T00:48:39,488 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:39,489 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:39,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-15T00:48:39,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:48:39,496 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:39,497 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:39,497 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:39,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:48:39,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-15T00:48:39,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:39,652 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T00:48:39,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/7c3e90f41806471cbeeebb21f75898dc is 50, key is test_row_0/A:col10/1731631717779/Put/seqid=0 2024-11-15T00:48:39,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741879_1055 (size=12151) 2024-11-15T00:48:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:48:39,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:39,983 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:39,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:39,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631779988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:39,993 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:39,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631779990, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:39,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631779993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:39,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631779993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631780019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,026 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 5b99d5a4d9ee135a84ccb828f4037b20 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2206 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:40,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,028 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,099 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631780097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,100 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,106 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/7c3e90f41806471cbeeebb21f75898dc 2024-11-15T00:48:40,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631780109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631780109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631780110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,112 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,113 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:48:40,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/1099d2c1b9a446a0816afd89b0ee1087 is 50, key is test_row_0/B:col10/1731631717779/Put/seqid=0 2024-11-15T00:48:40,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741880_1056 (size=12151) 2024-11-15T00:48:40,191 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/1099d2c1b9a446a0816afd89b0ee1087 2024-11-15T00:48:40,210 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/380fb3718f2c47df945be2d6e746ab01 is 50, key is test_row_0/C:col10/1731631717779/Put/seqid=0 2024-11-15T00:48:40,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741881_1057 (size=12151) 2024-11-15T00:48:40,233 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=232 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/380fb3718f2c47df945be2d6e746ab01 2024-11-15T00:48:40,241 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/7c3e90f41806471cbeeebb21f75898dc as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/7c3e90f41806471cbeeebb21f75898dc 2024-11-15T00:48:40,256 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/7c3e90f41806471cbeeebb21f75898dc, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T00:48:40,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/1099d2c1b9a446a0816afd89b0ee1087 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1099d2c1b9a446a0816afd89b0ee1087 2024-11-15T00:48:40,267 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1099d2c1b9a446a0816afd89b0ee1087, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T00:48:40,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/380fb3718f2c47df945be2d6e746ab01 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/380fb3718f2c47df945be2d6e746ab01 2024-11-15T00:48:40,298 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/380fb3718f2c47df945be2d6e746ab01, entries=150, sequenceid=232, filesize=11.9 K 2024-11-15T00:48:40,300 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for 5b99d5a4d9ee135a84ccb828f4037b20 in 647ms, sequenceid=232, compaction requested=true 2024-11-15T00:48:40,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:40,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:40,300 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-15T00:48:40,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-15T00:48:40,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-15T00:48:40,329 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 816 msec 2024-11-15T00:48:40,338 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 847 msec 2024-11-15T00:48:40,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:40,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:40,379 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T00:48:40,392 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/f1ddbb18c3e447d795e7e4eeb74a5ddc is 50, key is test_row_0/A:col10/1731631720314/Put/seqid=0 2024-11-15T00:48:40,439 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741882_1058 (size=21715) 2024-11-15T00:48:40,441 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/f1ddbb18c3e447d795e7e4eeb74a5ddc 2024-11-15T00:48:40,475 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/ef95442ad70541118f6e237c000da1b7 is 50, key is test_row_0/B:col10/1731631720314/Put/seqid=0 2024-11-15T00:48:40,493 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631780482, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,494 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,497 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631780486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,498 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,498 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631780486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,499 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631780489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,500 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741883_1059 (size=12151) 2024-11-15T00:48:40,601 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631780597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,602 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631780607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,611 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,611 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631780607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,612 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,612 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631780609, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,622 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,623 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-15T00:48:40,627 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:40,628 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:40,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-15T00:48:40,633 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:40,634 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:40,634 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:40,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:48:40,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:48:40,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T00:48:40,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:40,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:40,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:40,798 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:40,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:40,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:40,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631780812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631780817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,825 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631780820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,829 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,833 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:40,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631780832, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:40,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:40,836 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:40,929 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/ef95442ad70541118f6e237c000da1b7 2024-11-15T00:48:40,957 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T00:48:40,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:40,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:40,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:40,957 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:40,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:48:40,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:40,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/6856843c41804b188fd4c641b9935eb5 is 50, key is test_row_0/C:col10/1731631720314/Put/seqid=0 2024-11-15T00:48:40,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741884_1060 (size=12151) 2024-11-15T00:48:41,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T00:48:41,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:41,124 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,124 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:41,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:41,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:41,143 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631781137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,144 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631781137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,146 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631781137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,150 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,151 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,152 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631781151, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,153 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:48:41,285 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T00:48:41,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:41,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,285 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:41,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:41,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:41,400 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/6856843c41804b188fd4c641b9935eb5 2024-11-15T00:48:41,413 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/f1ddbb18c3e447d795e7e4eeb74a5ddc as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/f1ddbb18c3e447d795e7e4eeb74a5ddc 2024-11-15T00:48:41,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/f1ddbb18c3e447d795e7e4eeb74a5ddc, entries=350, sequenceid=245, filesize=21.2 K 2024-11-15T00:48:41,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/ef95442ad70541118f6e237c000da1b7 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/ef95442ad70541118f6e237c000da1b7 2024-11-15T00:48:41,434 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/ef95442ad70541118f6e237c000da1b7, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T00:48:41,445 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T00:48:41,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:41,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,446 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:41,446 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:41,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/6856843c41804b188fd4c641b9935eb5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6856843c41804b188fd4c641b9935eb5 2024-11-15T00:48:41,468 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6856843c41804b188fd4c641b9935eb5, entries=150, sequenceid=245, filesize=11.9 K 2024-11-15T00:48:41,473 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1094ms, sequenceid=245, compaction requested=true 2024-11-15T00:48:41,473 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:41,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:41,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:41,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:41,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:48:41,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:41,473 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T00:48:41,474 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T00:48:41,476 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 6 store files, 0 compacting, 6 eligible, 16 blocking 2024-11-15T00:48:41,476 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 82812 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T00:48:41,477 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/A is initiating minor compaction (all files) 2024-11-15T00:48:41,477 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/A in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,477 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/9af89a258442452687475e35e96c0743, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b967f501065a4ea08636e0390fc93a0b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/32208904cbc745f8860dab4737b6b0ec, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/1b20fccf80474ff6be84695122d16556, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/7c3e90f41806471cbeeebb21f75898dc, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/f1ddbb18c3e447d795e7e4eeb74a5ddc] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=80.9 K 2024-11-15T00:48:41,477 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9af89a258442452687475e35e96c0743, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631712699 2024-11-15T00:48:41,478 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b967f501065a4ea08636e0390fc93a0b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731631712857 2024-11-15T00:48:41,478 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 32208904cbc745f8860dab4737b6b0ec, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731631714144 2024-11-15T00:48:41,479 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b20fccf80474ff6be84695122d16556, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731631715397 2024-11-15T00:48:41,479 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7c3e90f41806471cbeeebb21f75898dc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731631717779 2024-11-15T00:48:41,479 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 6 files of size 73248 starting at candidate #0 after considering 10 permutations with 10 in ratio 2024-11-15T00:48:41,480 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/C is initiating minor compaction (all files) 2024-11-15T00:48:41,480 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/C in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,480 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/66a249b74bbd42d3883d132171e00f31, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5aaf5c32b340489f884c294f3888b094, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/7e5e56adffd44e87a842d7d35d78cb6c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/caf12d168bce464281c8f54b9f358a16, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/380fb3718f2c47df945be2d6e746ab01, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6856843c41804b188fd4c641b9935eb5] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=71.5 K 2024-11-15T00:48:41,480 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f1ddbb18c3e447d795e7e4eeb74a5ddc, keycount=350, bloomtype=ROW, size=21.2 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631719989 2024-11-15T00:48:41,481 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 66a249b74bbd42d3883d132171e00f31, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631712699 2024-11-15T00:48:41,481 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5aaf5c32b340489f884c294f3888b094, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731631712857 2024-11-15T00:48:41,482 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7e5e56adffd44e87a842d7d35d78cb6c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731631714144 2024-11-15T00:48:41,483 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting caf12d168bce464281c8f54b9f358a16, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731631715397 2024-11-15T00:48:41,484 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 380fb3718f2c47df945be2d6e746ab01, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731631717779 2024-11-15T00:48:41,485 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6856843c41804b188fd4c641b9935eb5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631720314 2024-11-15T00:48:41,517 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#A#compaction#48 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:41,518 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/52b972123c1143c096582c79d2fb4e96 is 50, key is test_row_0/A:col10/1731631720314/Put/seqid=0 2024-11-15T00:48:41,528 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#C#compaction#49 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:41,529 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/188e586636da4f92bbc0af66ab7962b8 is 50, key is test_row_0/C:col10/1731631720314/Put/seqid=0 2024-11-15T00:48:41,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741885_1061 (size=12697) 2024-11-15T00:48:41,590 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741886_1062 (size=12697) 2024-11-15T00:48:41,604 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-15T00:48:41,608 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,608 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T00:48:41,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/8769462fd8e745b9a3704202c3ba384e is 50, key is test_row_0/A:col10/1731631720485/Put/seqid=0 2024-11-15T00:48:41,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741887_1063 (size=12301) 2024-11-15T00:48:41,626 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/8769462fd8e745b9a3704202c3ba384e 2024-11-15T00:48:41,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/bf84628124044ad195536daf4dcff845 is 50, key is test_row_0/B:col10/1731631720485/Put/seqid=0 2024-11-15T00:48:41,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:41,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:41,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741888_1064 (size=12301) 2024-11-15T00:48:41,690 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631781678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,690 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/bf84628124044ad195536daf4dcff845 2024-11-15T00:48:41,690 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631781690, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631781691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,702 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,702 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631781692, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,703 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/4ea022b61bad4d019f32c01b83d36c94 is 50, key is test_row_0/C:col10/1731631720485/Put/seqid=0 2024-11-15T00:48:41,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741889_1065 (size=12301) 2024-11-15T00:48:41,734 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=268 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/4ea022b61bad4d019f32c01b83d36c94 2024-11-15T00:48:41,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/8769462fd8e745b9a3704202c3ba384e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8769462fd8e745b9a3704202c3ba384e 2024-11-15T00:48:41,748 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8769462fd8e745b9a3704202c3ba384e, entries=150, sequenceid=268, filesize=12.0 K 2024-11-15T00:48:41,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/bf84628124044ad195536daf4dcff845 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/bf84628124044ad195536daf4dcff845 2024-11-15T00:48:41,762 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/bf84628124044ad195536daf4dcff845, entries=150, sequenceid=268, filesize=12.0 K 2024-11-15T00:48:41,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/4ea022b61bad4d019f32c01b83d36c94 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4ea022b61bad4d019f32c01b83d36c94 2024-11-15T00:48:41,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:48:41,782 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4ea022b61bad4d019f32c01b83d36c94, entries=150, sequenceid=268, filesize=12.0 K 2024-11-15T00:48:41,785 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 5b99d5a4d9ee135a84ccb828f4037b20 in 177ms, sequenceid=268, compaction requested=true 2024-11-15T00:48:41,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:41,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-15T00:48:41,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-15T00:48:41,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-15T00:48:41,792 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1530 sec 2024-11-15T00:48:41,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:41,800 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:41,800 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T00:48:41,800 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 1.1700 sec 2024-11-15T00:48:41,809 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/49815c89cb4d47e188f1c01940df73c2 is 50, key is test_row_0/A:col10/1731631721798/Put/seqid=0 2024-11-15T00:48:41,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741890_1066 (size=14741) 2024-11-15T00:48:41,834 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/49815c89cb4d47e188f1c01940df73c2 2024-11-15T00:48:41,846 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/1976ccb79d264d9180f059c831fc0675 is 50, key is test_row_0/B:col10/1731631721798/Put/seqid=0 2024-11-15T00:48:41,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631781854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,866 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631781862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,874 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,875 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631781864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,876 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:41,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741891_1067 (size=12301) 2024-11-15T00:48:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631781865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,876 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:41,877 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:41,877 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/1976ccb79d264d9180f059c831fc0675 2024-11-15T00:48:41,898 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/dc6f499212034273a61532fe42e700c6 is 50, key is test_row_0/C:col10/1731631721798/Put/seqid=0 2024-11-15T00:48:41,910 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741892_1068 (size=12301) 2024-11-15T00:48:41,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=283 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/dc6f499212034273a61532fe42e700c6 2024-11-15T00:48:41,919 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/49815c89cb4d47e188f1c01940df73c2 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/49815c89cb4d47e188f1c01940df73c2 2024-11-15T00:48:41,926 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/49815c89cb4d47e188f1c01940df73c2, entries=200, sequenceid=283, filesize=14.4 K 2024-11-15T00:48:41,928 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/1976ccb79d264d9180f059c831fc0675 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1976ccb79d264d9180f059c831fc0675 2024-11-15T00:48:41,936 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1976ccb79d264d9180f059c831fc0675, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T00:48:41,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/dc6f499212034273a61532fe42e700c6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/dc6f499212034273a61532fe42e700c6 2024-11-15T00:48:41,951 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/dc6f499212034273a61532fe42e700c6, entries=150, sequenceid=283, filesize=12.0 K 2024-11-15T00:48:41,952 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5b99d5a4d9ee135a84ccb828f4037b20 in 152ms, sequenceid=283, compaction requested=true 2024-11-15T00:48:41,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:41,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:A, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:41,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:48:41,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:B, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:41,952 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T00:48:41,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:41,953 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T00:48:41,969 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/52b972123c1143c096582c79d2fb4e96 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/52b972123c1143c096582c79d2fb4e96 2024-11-15T00:48:41,982 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/A of 5b99d5a4d9ee135a84ccb828f4037b20 into 52b972123c1143c096582c79d2fb4e96(size=12.4 K), total size for store is 38.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:41,982 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:41,982 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/A, priority=10, startTime=1731631721473; duration=0sec 2024-11-15T00:48:41,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:41,982 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:41,982 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T00:48:41,982 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:41,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T00:48:41,982 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:41,983 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 6 compacting, 2 eligible, 16 blocking 2024-11-15T00:48:41,984 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:48:41,984 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:48:41,984 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:41,984 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:C 2024-11-15T00:48:41,984 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 0 compacting, 8 eligible, 16 blocking 2024-11-15T00:48:41,992 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 8 files of size 97850 starting at candidate #0 after considering 21 permutations with 21 in ratio 2024-11-15T00:48:41,992 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/B is initiating minor compaction (all files) 2024-11-15T00:48:41,992 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/B in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:41,993 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2a6c1ec6d3104b6ebd4025c3086ac56c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/fdc88f51de814ee2af46bbd046c144c4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/966c910f54324219a8fbc87ec0ba0d75, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/0a576c2f8d4749a7b1415653004341ac, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1099d2c1b9a446a0816afd89b0ee1087, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/ef95442ad70541118f6e237c000da1b7, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/bf84628124044ad195536daf4dcff845, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1976ccb79d264d9180f059c831fc0675] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=95.6 K 2024-11-15T00:48:41,994 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2a6c1ec6d3104b6ebd4025c3086ac56c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631712699 2024-11-15T00:48:41,994 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/d6b66ce70d0d4b5fb4834b0652c596df is 50, key is test_row_0/A:col10/1731631721980/Put/seqid=0 2024-11-15T00:48:41,995 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting fdc88f51de814ee2af46bbd046c144c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1731631712857 2024-11-15T00:48:41,997 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 966c910f54324219a8fbc87ec0ba0d75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731631714144 2024-11-15T00:48:41,997 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0a576c2f8d4749a7b1415653004341ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1731631715397 2024-11-15T00:48:41,998 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1099d2c1b9a446a0816afd89b0ee1087, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=232, earliestPutTs=1731631717779 2024-11-15T00:48:41,998 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ef95442ad70541118f6e237c000da1b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631720314 2024-11-15T00:48:42,003 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/188e586636da4f92bbc0af66ab7962b8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/188e586636da4f92bbc0af66ab7962b8 2024-11-15T00:48:42,002 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting bf84628124044ad195536daf4dcff845, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731631720469 2024-11-15T00:48:42,004 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1976ccb79d264d9180f059c831fc0675, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731631721684 2024-11-15T00:48:42,018 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 6 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/C of 5b99d5a4d9ee135a84ccb828f4037b20 into 188e586636da4f92bbc0af66ab7962b8(size=12.4 K), total size for store is 36.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:42,018 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:42,018 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/C, priority=10, startTime=1731631721473; duration=0sec 2024-11-15T00:48:42,018 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:48:42,018 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:C 2024-11-15T00:48:42,018 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 8 store files, 8 compacting, 0 eligible, 16 blocking 2024-11-15T00:48:42,019 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:48:42,019 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:48:42,019 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:42,019 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:B 2024-11-15T00:48:42,019 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:42,022 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39739 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:42,022 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/A is initiating minor compaction (all files) 2024-11-15T00:48:42,022 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/A in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:42,023 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/52b972123c1143c096582c79d2fb4e96, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8769462fd8e745b9a3704202c3ba384e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/49815c89cb4d47e188f1c01940df73c2] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=38.8 K 2024-11-15T00:48:42,023 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 52b972123c1143c096582c79d2fb4e96, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631720314 2024-11-15T00:48:42,024 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8769462fd8e745b9a3704202c3ba384e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731631720469 2024-11-15T00:48:42,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631782011, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,027 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631782020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,028 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,029 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,029 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49815c89cb4d47e188f1c01940df73c2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731631721671 2024-11-15T00:48:42,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631782027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,031 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741893_1069 (size=12301) 2024-11-15T00:48:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,033 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631782029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,043 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#A#compaction#57 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:42,044 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/4c775c2157074d26b8a34b0d5c7f1eb6 is 50, key is test_row_0/A:col10/1731631721798/Put/seqid=0 2024-11-15T00:48:42,048 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#B#compaction#58 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:42,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57598 deadline: 1731631782046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,048 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/c7139b26c12348c282798236d720ba77 is 50, key is test_row_0/B:col10/1731631721798/Put/seqid=0 2024-11-15T00:48:42,049 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 5b99d5a4d9ee135a84ccb828f4037b20 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,049 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741894_1070 (size=12949) 2024-11-15T00:48:42,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741895_1071 (size=12915) 2024-11-15T00:48:42,139 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631782137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,140 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631782138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,141 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631782138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,142 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631782138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,143 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631782348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631782348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631782348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,350 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631782348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,434 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/d6b66ce70d0d4b5fb4834b0652c596df 2024-11-15T00:48:42,469 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/82cb531b5bc344c88b801b43b7b99d27 is 50, key is test_row_0/B:col10/1731631721980/Put/seqid=0 2024-11-15T00:48:42,479 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/c7139b26c12348c282798236d720ba77 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/c7139b26c12348c282798236d720ba77 2024-11-15T00:48:42,493 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 8 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/B of 5b99d5a4d9ee135a84ccb828f4037b20 into c7139b26c12348c282798236d720ba77(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:42,493 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:42,494 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/B, priority=8, startTime=1731631721952; duration=0sec 2024-11-15T00:48:42,494 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:42,494 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:B 2024-11-15T00:48:42,500 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/4c775c2157074d26b8a34b0d5c7f1eb6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/4c775c2157074d26b8a34b0d5c7f1eb6 2024-11-15T00:48:42,503 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741896_1072 (size=12301) 2024-11-15T00:48:42,504 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/82cb531b5bc344c88b801b43b7b99d27 2024-11-15T00:48:42,536 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/A of 5b99d5a4d9ee135a84ccb828f4037b20 into 4c775c2157074d26b8a34b0d5c7f1eb6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:42,536 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:42,536 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/A, priority=13, startTime=1731631721952; duration=0sec 2024-11-15T00:48:42,536 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:42,536 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:42,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/6b68ea9bbff64b2c84f1b48574d8f261 is 50, key is test_row_0/C:col10/1731631721980/Put/seqid=0 2024-11-15T00:48:42,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741897_1073 (size=12301) 2024-11-15T00:48:42,571 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=307 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/6b68ea9bbff64b2c84f1b48574d8f261 2024-11-15T00:48:42,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/d6b66ce70d0d4b5fb4834b0652c596df as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d6b66ce70d0d4b5fb4834b0652c596df 2024-11-15T00:48:42,609 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d6b66ce70d0d4b5fb4834b0652c596df, entries=150, sequenceid=307, filesize=12.0 K 2024-11-15T00:48:42,611 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/82cb531b5bc344c88b801b43b7b99d27 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/82cb531b5bc344c88b801b43b7b99d27 2024-11-15T00:48:42,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/82cb531b5bc344c88b801b43b7b99d27, entries=150, sequenceid=307, filesize=12.0 K 2024-11-15T00:48:42,623 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/6b68ea9bbff64b2c84f1b48574d8f261 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6b68ea9bbff64b2c84f1b48574d8f261 2024-11-15T00:48:42,632 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6b68ea9bbff64b2c84f1b48574d8f261, entries=150, sequenceid=307, filesize=12.0 K 2024-11-15T00:48:42,634 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 5b99d5a4d9ee135a84ccb828f4037b20 in 652ms, sequenceid=307, compaction requested=true 2024-11-15T00:48:42,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:42,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:42,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:42,634 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T00:48:42,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:42,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:42,634 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T00:48:42,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:42,634 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:42,635 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:48:42,635 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:48:42,635 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:48:42,635 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:48:42,635 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:42,635 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:42,635 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:42,635 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:B 2024-11-15T00:48:42,635 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:48:42,636 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49600 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:48:42,637 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/C is initiating minor compaction (all files) 2024-11-15T00:48:42,637 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/C in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:42,637 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/188e586636da4f92bbc0af66ab7962b8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4ea022b61bad4d019f32c01b83d36c94, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/dc6f499212034273a61532fe42e700c6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6b68ea9bbff64b2c84f1b48574d8f261] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=48.4 K 2024-11-15T00:48:42,639 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 188e586636da4f92bbc0af66ab7962b8, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1731631720314 2024-11-15T00:48:42,639 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4ea022b61bad4d019f32c01b83d36c94, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=268, earliestPutTs=1731631720469 2024-11-15T00:48:42,640 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting dc6f499212034273a61532fe42e700c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731631721684 2024-11-15T00:48:42,640 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b68ea9bbff64b2c84f1b48574d8f261, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731631721863 2024-11-15T00:48:42,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:42,661 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:42,661 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T00:48:42,669 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#C#compaction#61 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:42,670 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/ee6322350da84474855fcc1cf8f4524f is 50, key is test_row_0/C:col10/1731631721980/Put/seqid=0 2024-11-15T00:48:42,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/ca0917d8a49c409a983b80a1d56168b8 is 50, key is test_row_0/A:col10/1731631722004/Put/seqid=0 2024-11-15T00:48:42,730 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631782726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631782721, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,731 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631782731, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741899_1075 (size=14741) 2024-11-15T00:48:42,735 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/ca0917d8a49c409a983b80a1d56168b8 2024-11-15T00:48:42,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741898_1074 (size=12983) 2024-11-15T00:48:42,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,740 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,741 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631782741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,745 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,758 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/ee6322350da84474855fcc1cf8f4524f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/ee6322350da84474855fcc1cf8f4524f 2024-11-15T00:48:42,778 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/b80a6646063940f497c67b25b562b43f is 50, key is test_row_0/B:col10/1731631722004/Put/seqid=0 2024-11-15T00:48:42,785 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/C of 5b99d5a4d9ee135a84ccb828f4037b20 into ee6322350da84474855fcc1cf8f4524f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:42,785 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:42,785 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/C, priority=12, startTime=1731631722634; duration=0sec 2024-11-15T00:48:42,785 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:42,785 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:C 2024-11-15T00:48:42,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-15T00:48:42,787 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:42,788 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:42,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-15T00:48:42,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T00:48:42,791 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:42,792 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:42,792 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:42,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741900_1076 (size=12301) 2024-11-15T00:48:42,840 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631782837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,841 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,841 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631782837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,842 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631782856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:42,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631782857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,859 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:42,864 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:42,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T00:48:42,945 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:42,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:42,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:42,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:42,946 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:42,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:42,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631783047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,049 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,050 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631783047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,051 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,074 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631783068, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,076 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631783072, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,077 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,078 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T00:48:43,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,231 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/b80a6646063940f497c67b25b562b43f 2024-11-15T00:48:43,253 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/4a2374877ffe4bc1be40e03b1868cd54 is 50, key is test_row_0/C:col10/1731631722004/Put/seqid=0 2024-11-15T00:48:43,267 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:43,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:43,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,291 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741901_1077 (size=12301) 2024-11-15T00:48:43,292 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/4a2374877ffe4bc1be40e03b1868cd54 2024-11-15T00:48:43,302 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/ca0917d8a49c409a983b80a1d56168b8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/ca0917d8a49c409a983b80a1d56168b8 2024-11-15T00:48:43,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/ca0917d8a49c409a983b80a1d56168b8, entries=200, sequenceid=325, filesize=14.4 K 2024-11-15T00:48:43,311 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/b80a6646063940f497c67b25b562b43f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/b80a6646063940f497c67b25b562b43f 2024-11-15T00:48:43,318 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/b80a6646063940f497c67b25b562b43f, entries=150, sequenceid=325, filesize=12.0 K 2024-11-15T00:48:43,320 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/4a2374877ffe4bc1be40e03b1868cd54 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4a2374877ffe4bc1be40e03b1868cd54 2024-11-15T00:48:43,328 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4a2374877ffe4bc1be40e03b1868cd54, entries=150, sequenceid=325, filesize=12.0 K 2024-11-15T00:48:43,329 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 5b99d5a4d9ee135a84ccb828f4037b20 in 668ms, sequenceid=325, compaction requested=true 2024-11-15T00:48:43,329 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:43,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:43,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:43,330 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:43,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:43,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:43,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:43,330 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:48:43,330 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:43,331 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:43,331 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/A is initiating minor compaction (all files) 2024-11-15T00:48:43,332 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/A in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,332 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/4c775c2157074d26b8a34b0d5c7f1eb6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d6b66ce70d0d4b5fb4834b0652c596df, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/ca0917d8a49c409a983b80a1d56168b8] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=39.1 K 2024-11-15T00:48:43,332 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:43,332 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/B is initiating minor compaction (all files) 2024-11-15T00:48:43,332 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/B in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,332 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/c7139b26c12348c282798236d720ba77, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/82cb531b5bc344c88b801b43b7b99d27, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/b80a6646063940f497c67b25b562b43f] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=36.6 K 2024-11-15T00:48:43,333 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c775c2157074d26b8a34b0d5c7f1eb6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731631721684 2024-11-15T00:48:43,335 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d6b66ce70d0d4b5fb4834b0652c596df, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731631721863 2024-11-15T00:48:43,335 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c7139b26c12348c282798236d720ba77, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=283, earliestPutTs=1731631721684 2024-11-15T00:48:43,336 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca0917d8a49c409a983b80a1d56168b8, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631722004 2024-11-15T00:48:43,336 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 82cb531b5bc344c88b801b43b7b99d27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731631721863 2024-11-15T00:48:43,337 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b80a6646063940f497c67b25b562b43f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631722004 2024-11-15T00:48:43,349 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#A#compaction#65 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:43,350 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/a58a75ad25da4f9288d764a6706913c0 is 50, key is test_row_0/A:col10/1731631722004/Put/seqid=0 2024-11-15T00:48:43,358 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#B#compaction#66 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:43,359 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/8aa5d5b1d6a749f3a61cb7f0a89027e0 is 50, key is test_row_0/B:col10/1731631722004/Put/seqid=0 2024-11-15T00:48:43,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:43,361 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:43,362 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T00:48:43,399 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631783397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,402 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,403 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631783398, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,404 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,406 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631783400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,409 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741902_1078 (size=13051) 2024-11-15T00:48:43,412 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/a3d8addfc5564e1db2cfff755da56413 is 50, key is test_row_0/A:col10/1731631722715/Put/seqid=0 2024-11-15T00:48:43,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631783399, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,414 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,415 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T00:48:43,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:43,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:43,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741903_1079 (size=13017) 2024-11-15T00:48:43,443 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/8aa5d5b1d6a749f3a61cb7f0a89027e0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/8aa5d5b1d6a749f3a61cb7f0a89027e0 2024-11-15T00:48:43,452 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/B of 5b99d5a4d9ee135a84ccb828f4037b20 into 8aa5d5b1d6a749f3a61cb7f0a89027e0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:43,453 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:43,453 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/B, priority=13, startTime=1731631723330; duration=0sec 2024-11-15T00:48:43,453 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:43,453 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:B 2024-11-15T00:48:43,453 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T00:48:43,454 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741904_1080 (size=14741) 2024-11-15T00:48:43,455 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:48:43,455 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:48:43,455 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:43,455 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:C 2024-11-15T00:48:43,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/a3d8addfc5564e1db2cfff755da56413 2024-11-15T00:48:43,476 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/98f2c5452c274682b3e35446ad790912 is 50, key is test_row_0/B:col10/1731631722715/Put/seqid=0 2024-11-15T00:48:43,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741905_1081 (size=12301) 2024-11-15T00:48:43,493 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/98f2c5452c274682b3e35446ad790912 2024-11-15T00:48:43,504 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/5b54c737ca3b40a5aececa040a1d7104 is 50, key is test_row_0/C:col10/1731631722715/Put/seqid=0 2024-11-15T00:48:43,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631783507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631783507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,511 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741906_1082 (size=12301) 2024-11-15T00:48:43,522 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631783517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,522 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,528 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631783526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:43,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:43,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631783717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,722 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,722 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631783717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,723 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631783726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,729 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,740 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:43,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:43,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631783737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:43,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:43,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:43,741 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,859 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/a58a75ad25da4f9288d764a6706913c0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a58a75ad25da4f9288d764a6706913c0 2024-11-15T00:48:43,877 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/A of 5b99d5a4d9ee135a84ccb828f4037b20 into a58a75ad25da4f9288d764a6706913c0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:43,878 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:43,878 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/A, priority=13, startTime=1731631723330; duration=0sec 2024-11-15T00:48:43,878 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:43,878 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:43,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:43,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:43,895 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:43,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=348 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/5b54c737ca3b40a5aececa040a1d7104 2024-11-15T00:48:43,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T00:48:43,953 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/a3d8addfc5564e1db2cfff755da56413 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a3d8addfc5564e1db2cfff755da56413 2024-11-15T00:48:43,966 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a3d8addfc5564e1db2cfff755da56413, entries=200, sequenceid=348, filesize=14.4 K 2024-11-15T00:48:43,969 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/98f2c5452c274682b3e35446ad790912 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/98f2c5452c274682b3e35446ad790912 2024-11-15T00:48:43,984 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/98f2c5452c274682b3e35446ad790912, entries=150, sequenceid=348, filesize=12.0 K 2024-11-15T00:48:43,985 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/5b54c737ca3b40a5aececa040a1d7104 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5b54c737ca3b40a5aececa040a1d7104 2024-11-15T00:48:44,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5b54c737ca3b40a5aececa040a1d7104, entries=150, sequenceid=348, filesize=12.0 K 2024-11-15T00:48:44,002 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 5b99d5a4d9ee135a84ccb828f4037b20 in 641ms, sequenceid=348, compaction requested=true 2024-11-15T00:48:44,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:44,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:44,003 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T00:48:44,003 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:44,003 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T00:48:44,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:44,005 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:48:44,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:44,005 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:48:44,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:44,005 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:44,005 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:44,005 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:44,005 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:44,006 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:48:44,006 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:48:44,006 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:44,006 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:B 2024-11-15T00:48:44,008 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:44,009 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/C is initiating minor compaction (all files) 2024-11-15T00:48:44,010 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/C in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,010 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/ee6322350da84474855fcc1cf8f4524f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4a2374877ffe4bc1be40e03b1868cd54, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5b54c737ca3b40a5aececa040a1d7104] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=36.7 K 2024-11-15T00:48:44,011 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee6322350da84474855fcc1cf8f4524f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=307, earliestPutTs=1731631721863 2024-11-15T00:48:44,012 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4a2374877ffe4bc1be40e03b1868cd54, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631722004 2024-11-15T00:48:44,014 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b54c737ca3b40a5aececa040a1d7104, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731631722715 2024-11-15T00:48:44,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:44,034 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:44,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T00:48:44,037 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#C#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:44,038 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/859db33a5b9f48868c23ffb211f1bbe3 is 50, key is test_row_0/C:col10/1731631722715/Put/seqid=0 2024-11-15T00:48:44,051 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/6c9e1e9a2a1a40638d2890257cfc39b6 is 50, key is test_row_0/A:col10/1731631723398/Put/seqid=0 2024-11-15T00:48:44,055 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:44,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:44,055 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,055 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741907_1083 (size=13085) 2024-11-15T00:48:44,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,065 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741908_1084 (size=14741) 2024-11-15T00:48:44,117 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631784113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,118 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,126 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631784118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,127 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,129 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631784125, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,131 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,132 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631784132, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,137 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,213 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:44,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:44,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,214 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,229 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631784226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,229 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,230 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631784237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,239 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631784237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,240 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,250 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631784247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,250 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:44,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:44,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,392 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,439 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631784437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,439 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,440 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,450 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631784447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631784447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631784458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,461 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,463 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/6c9e1e9a2a1a40638d2890257cfc39b6 2024-11-15T00:48:44,484 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/859db33a5b9f48868c23ffb211f1bbe3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/859db33a5b9f48868c23ffb211f1bbe3 2024-11-15T00:48:44,494 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/C of 5b99d5a4d9ee135a84ccb828f4037b20 into 859db33a5b9f48868c23ffb211f1bbe3(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:44,494 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:44,494 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/C, priority=13, startTime=1731631724005; duration=0sec 2024-11-15T00:48:44,494 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:44,495 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:C 2024-11-15T00:48:44,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/d2630ff4134c4add91c150e5f7f48d26 is 50, key is test_row_0/B:col10/1731631723398/Put/seqid=0 2024-11-15T00:48:44,546 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741909_1085 (size=12301) 2024-11-15T00:48:44,553 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/d2630ff4134c4add91c150e5f7f48d26 2024-11-15T00:48:44,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:44,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:44,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,558 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,558 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,578 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/3048a4c014f34ee1b89ee24afb662573 is 50, key is test_row_0/C:col10/1731631723398/Put/seqid=0 2024-11-15T00:48:44,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741910_1086 (size=12301) 2024-11-15T00:48:44,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:44,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:44,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,717 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,717 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631784747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,762 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631784760, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,764 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,761 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631784757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,777 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,777 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:44,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631784767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:44,786 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:44,787 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:44,871 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:44,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:44,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:44,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T00:48:45,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=366 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/3048a4c014f34ee1b89ee24afb662573 2024-11-15T00:48:45,027 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:45,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:45,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:45,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:45,028 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] handler.RSProcedureHandler(58): pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:45,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=26 java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:45,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=26 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:45,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/6c9e1e9a2a1a40638d2890257cfc39b6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6c9e1e9a2a1a40638d2890257cfc39b6 2024-11-15T00:48:45,042 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6c9e1e9a2a1a40638d2890257cfc39b6, entries=200, sequenceid=366, filesize=14.4 K 2024-11-15T00:48:45,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/d2630ff4134c4add91c150e5f7f48d26 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/d2630ff4134c4add91c150e5f7f48d26 2024-11-15T00:48:45,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/d2630ff4134c4add91c150e5f7f48d26, entries=150, sequenceid=366, filesize=12.0 K 2024-11-15T00:48:45,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/3048a4c014f34ee1b89ee24afb662573 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/3048a4c014f34ee1b89ee24afb662573 2024-11-15T00:48:45,137 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/3048a4c014f34ee1b89ee24afb662573, entries=150, sequenceid=366, filesize=12.0 K 2024-11-15T00:48:45,142 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=120.76 KB/123660 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1107ms, sequenceid=366, compaction requested=true 2024-11-15T00:48:45,142 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:45,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:45,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:45,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:45,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:45,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:45,143 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:45,145 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:45,145 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:45,149 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42533 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:45,149 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/A is initiating minor compaction (all files) 2024-11-15T00:48:45,149 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/A in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:45,150 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a58a75ad25da4f9288d764a6706913c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a3d8addfc5564e1db2cfff755da56413, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6c9e1e9a2a1a40638d2890257cfc39b6] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=41.5 K 2024-11-15T00:48:45,150 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:45,150 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a58a75ad25da4f9288d764a6706913c0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631722004 2024-11-15T00:48:45,150 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 5b99d5a4d9ee135a84ccb828f4037b20/B is initiating minor compaction (all files) 2024-11-15T00:48:45,150 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 5b99d5a4d9ee135a84ccb828f4037b20/B in TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:45,150 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/8aa5d5b1d6a749f3a61cb7f0a89027e0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/98f2c5452c274682b3e35446ad790912, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/d2630ff4134c4add91c150e5f7f48d26] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp, totalSize=36.7 K 2024-11-15T00:48:45,151 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3d8addfc5564e1db2cfff755da56413, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731631722715 2024-11-15T00:48:45,151 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8aa5d5b1d6a749f3a61cb7f0a89027e0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631722004 2024-11-15T00:48:45,151 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c9e1e9a2a1a40638d2890257cfc39b6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731631723393 2024-11-15T00:48:45,152 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 98f2c5452c274682b3e35446ad790912, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=348, earliestPutTs=1731631722715 2024-11-15T00:48:45,153 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d2630ff4134c4add91c150e5f7f48d26, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=366, earliestPutTs=1731631723398 2024-11-15T00:48:45,175 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#A#compaction#74 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:45,176 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/fa4bb4c54f7e47cf84ab3ca34aedd173 is 50, key is test_row_0/A:col10/1731631723398/Put/seqid=0 2024-11-15T00:48:45,178 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 5b99d5a4d9ee135a84ccb828f4037b20#B#compaction#75 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:45,179 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/2e8b5a83cfaf4175bc18d71149219700 is 50, key is test_row_0/B:col10/1731631723398/Put/seqid=0 2024-11-15T00:48:45,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-15T00:48:45,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:45,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T00:48:45,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/ab168a4442ed416ea32c1343b5388f12 is 50, key is test_row_0/A:col10/1731631724115/Put/seqid=0 2024-11-15T00:48:45,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741912_1088 (size=13119) 2024-11-15T00:48:45,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741911_1087 (size=13153) 2024-11-15T00:48:45,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741913_1089 (size=12301) 2024-11-15T00:48:45,256 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/ab168a4442ed416ea32c1343b5388f12 2024-11-15T00:48:45,260 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/fa4bb4c54f7e47cf84ab3ca34aedd173 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/fa4bb4c54f7e47cf84ab3ca34aedd173 2024-11-15T00:48:45,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. as already flushing 2024-11-15T00:48:45,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:45,275 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/A of 5b99d5a4d9ee135a84ccb828f4037b20 into fa4bb4c54f7e47cf84ab3ca34aedd173(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:45,275 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:45,275 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/A, priority=13, startTime=1731631725142; duration=0sec 2024-11-15T00:48:45,279 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/0c67bfb6ef5740fc98486adb90f63ed3 is 50, key is test_row_0/B:col10/1731631724115/Put/seqid=0 2024-11-15T00:48:45,275 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:45,279 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:45,280 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T00:48:45,282 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:48:45,282 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:48:45,282 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:45,282 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:C 2024-11-15T00:48:45,320 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741914_1090 (size=12301) 2024-11-15T00:48:45,321 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/0c67bfb6ef5740fc98486adb90f63ed3 2024-11-15T00:48:45,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631785323, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,330 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,333 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631785328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631785330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,335 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631785331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,336 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,343 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-15T00:48:45,343 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-15T00:48:45,343 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:48:45,343 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:48:45,349 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,349 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,349 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-15T00:48:45,349 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:48:45,349 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,349 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-15T00:48:45,349 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,349 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,350 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:48:45,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:48:45,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:48:45,350 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,350 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:48:45,350 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,350 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:48:45,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/c634fdc90e4d42a59baeecaa7e9763fb is 50, key is test_row_0/C:col10/1731631724115/Put/seqid=0 2024-11-15T00:48:45,376 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741915_1091 (size=12301) 2024-11-15T00:48:45,437 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631785437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,438 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631785448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631785448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,449 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,450 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631785450, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,451 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,452 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,626 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/2e8b5a83cfaf4175bc18d71149219700 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2e8b5a83cfaf4175bc18d71149219700 2024-11-15T00:48:45,633 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 5b99d5a4d9ee135a84ccb828f4037b20/B of 5b99d5a4d9ee135a84ccb828f4037b20 into 2e8b5a83cfaf4175bc18d71149219700(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:45,634 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:45,634 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., storeName=5b99d5a4d9ee135a84ccb828f4037b20/B, priority=13, startTime=1731631725142; duration=0sec 2024-11-15T00:48:45,634 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:45,634 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:B 2024-11-15T00:48:45,647 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57600 deadline: 1731631785647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57582 deadline: 1731631785657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57580 deadline: 1731631785657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:57546 deadline: 1731631785657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,658 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:45,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:48:45,777 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/c634fdc90e4d42a59baeecaa7e9763fb 2024-11-15T00:48:45,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/ab168a4442ed416ea32c1343b5388f12 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/ab168a4442ed416ea32c1343b5388f12 2024-11-15T00:48:45,790 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/ab168a4442ed416ea32c1343b5388f12, entries=150, sequenceid=388, filesize=12.0 K 2024-11-15T00:48:45,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/0c67bfb6ef5740fc98486adb90f63ed3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/0c67bfb6ef5740fc98486adb90f63ed3 2024-11-15T00:48:45,799 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/0c67bfb6ef5740fc98486adb90f63ed3, entries=150, sequenceid=388, filesize=12.0 K 2024-11-15T00:48:45,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/c634fdc90e4d42a59baeecaa7e9763fb as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c634fdc90e4d42a59baeecaa7e9763fb 2024-11-15T00:48:45,807 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c634fdc90e4d42a59baeecaa7e9763fb, entries=150, sequenceid=388, filesize=12.0 K 2024-11-15T00:48:45,808 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 5b99d5a4d9ee135a84ccb828f4037b20 in 623ms, sequenceid=388, compaction requested=true 2024-11-15T00:48:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:45,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-15T00:48:45,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-15T00:48:45,813 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-15T00:48:45,814 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0180 sec 2024-11-15T00:48:45,816 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 3.0260 sec 2024-11-15T00:48:45,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:45,959 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:45,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T00:48:45,959 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-15T00:48:45,959 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:48:45,959 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,959 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,959 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:48:45,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/a0e691148e5546a2b6a9223b5b4642d8 is 50, key is test_row_0/A:col10/1731631725327/Put/seqid=0 2024-11-15T00:48:45,968 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741916_1092 (size=12301) 2024-11-15T00:48:45,968 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-15T00:48:45,968 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-15T00:48:45,968 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-15T00:48:45,968 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:48:45,968 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:48:45,968 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:48:45,968 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,968 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,968 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,968 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,968 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,968 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:45,968 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:48:45,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:48:45,969 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:48:46,058 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-15T00:48:46,058 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:48:46,059 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:46,059 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:46,059 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:48:46,368 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/a0e691148e5546a2b6a9223b5b4642d8 2024-11-15T00:48:46,376 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/7c7e4dfe55244f50967446e066e25baf is 50, key is test_row_0/B:col10/1731631725327/Put/seqid=0 2024-11-15T00:48:46,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741917_1093 (size=12301) 2024-11-15T00:48:46,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/7c7e4dfe55244f50967446e066e25baf 2024-11-15T00:48:46,787 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:48:46,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/2a3531aec3584bdba260319d95cb8418 is 50, key is test_row_0/C:col10/1731631725327/Put/seqid=0 2024-11-15T00:48:46,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741918_1094 (size=12301) 2024-11-15T00:48:46,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-15T00:48:46,957 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:46,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:48:46,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-15T00:48:46,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-15T00:48:46,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-15T00:48:46,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 72 2024-11-15T00:48:46,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 73 2024-11-15T00:48:46,957 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:48:46,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1980 2024-11-15T00:48:46,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 1932 2024-11-15T00:48:46,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:48:46,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 976 2024-11-15T00:48:46,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2914 rows 2024-11-15T00:48:46,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 981 2024-11-15T00:48:46,958 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2932 rows 2024-11-15T00:48:46,958 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:48:46,958 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:48:46,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:46,958 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:46,958 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:48:46,963 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:48:46,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:48:46,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T00:48:46,970 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631726970"}]},"ts":"1731631726970"} 2024-11-15T00:48:46,972 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:48:46,972 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:48:46,974 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:48:46,981 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b99d5a4d9ee135a84ccb828f4037b20, UNASSIGN}] 2024-11-15T00:48:46,982 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b99d5a4d9ee135a84ccb828f4037b20, UNASSIGN 2024-11-15T00:48:46,984 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=5b99d5a4d9ee135a84ccb828f4037b20, regionState=CLOSING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:46,987 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=29, ppid=28, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b99d5a4d9ee135a84ccb828f4037b20, UNASSIGN because future has completed 2024-11-15T00:48:46,988 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:48:46,988 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:48:47,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T00:48:47,150 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(122): Close 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:47,150 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:48:47,150 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1722): Closing 5b99d5a4d9ee135a84ccb828f4037b20, disabling compactions & flushes 2024-11-15T00:48:47,150 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:47,203 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=406 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/2a3531aec3584bdba260319d95cb8418 2024-11-15T00:48:47,208 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/a0e691148e5546a2b6a9223b5b4642d8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a0e691148e5546a2b6a9223b5b4642d8 2024-11-15T00:48:47,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a0e691148e5546a2b6a9223b5b4642d8, entries=150, sequenceid=406, filesize=12.0 K 2024-11-15T00:48:47,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/7c7e4dfe55244f50967446e066e25baf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/7c7e4dfe55244f50967446e066e25baf 2024-11-15T00:48:47,219 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/7c7e4dfe55244f50967446e066e25baf, entries=150, sequenceid=406, filesize=12.0 K 2024-11-15T00:48:47,220 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/2a3531aec3584bdba260319d95cb8418 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2a3531aec3584bdba260319d95cb8418 2024-11-15T00:48:47,225 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2a3531aec3584bdba260319d95cb8418, entries=150, sequenceid=406, filesize=12.0 K 2024-11-15T00:48:47,226 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=26.84 KB/27480 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1267ms, sequenceid=406, compaction requested=true 2024-11-15T00:48:47,226 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 5b99d5a4d9ee135a84ccb828f4037b20: 2024-11-15T00:48:47,226 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:47,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:47,226 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:47,226 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:47,226 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. after waiting 0 ms 2024-11-15T00:48:47,226 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:47,226 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:A 2024-11-15T00:48:47,226 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:47,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:B, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:47,227 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:47,227 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:B 2024-11-15T00:48:47,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:47,227 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(2902): Flushing 5b99d5a4d9ee135a84ccb828f4037b20 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-15T00:48:47,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 5b99d5a4d9ee135a84ccb828f4037b20:C, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:47,227 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. because compaction request was cancelled 2024-11-15T00:48:47,227 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 5b99d5a4d9ee135a84ccb828f4037b20:C 2024-11-15T00:48:47,227 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:47,230 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/fd67c40d5a5f4c66861c76a2113cff2b is 50, key is test_row_0/A:col10/1731631726057/Put/seqid=0 2024-11-15T00:48:47,234 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741919_1095 (size=9857) 2024-11-15T00:48:47,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T00:48:47,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T00:48:47,635 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/fd67c40d5a5f4c66861c76a2113cff2b 2024-11-15T00:48:47,646 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/112e5ea7990949f6b84829b8f6511882 is 50, key is test_row_0/B:col10/1731631726057/Put/seqid=0 2024-11-15T00:48:47,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741920_1096 (size=9857) 2024-11-15T00:48:48,056 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/112e5ea7990949f6b84829b8f6511882 2024-11-15T00:48:48,064 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/f88bcacba07e4a2fa57e3cb9b6004328 is 50, key is test_row_0/C:col10/1731631726057/Put/seqid=0 2024-11-15T00:48:48,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741921_1097 (size=9857) 2024-11-15T00:48:48,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T00:48:48,469 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=413 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/f88bcacba07e4a2fa57e3cb9b6004328 2024-11-15T00:48:48,476 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/A/fd67c40d5a5f4c66861c76a2113cff2b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/fd67c40d5a5f4c66861c76a2113cff2b 2024-11-15T00:48:48,481 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/fd67c40d5a5f4c66861c76a2113cff2b, entries=100, sequenceid=413, filesize=9.6 K 2024-11-15T00:48:48,481 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/B/112e5ea7990949f6b84829b8f6511882 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/112e5ea7990949f6b84829b8f6511882 2024-11-15T00:48:48,486 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/112e5ea7990949f6b84829b8f6511882, entries=100, sequenceid=413, filesize=9.6 K 2024-11-15T00:48:48,487 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/.tmp/C/f88bcacba07e4a2fa57e3cb9b6004328 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/f88bcacba07e4a2fa57e3cb9b6004328 2024-11-15T00:48:48,492 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/f88bcacba07e4a2fa57e3cb9b6004328, entries=100, sequenceid=413, filesize=9.6 K 2024-11-15T00:48:48,493 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1267ms, sequenceid=413, compaction requested=true 2024-11-15T00:48:48,494 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/12ee2a5656d14d40ab286680006a3771, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/18f7532207124f05a772ac1ba99fd38b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b57cd07500d84854bdd4d08ddca21ebd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/90f731425b384928a5f7aa7634d94d6e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8ec1d437e57847ebab95abdd7993369b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d2446edcbe68451f972cadaf31191c66, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/09eb8a4a064c486baba88daf0cb86bbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/36ade1a1f41a40258aaf2efb5b756d83, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/018070c79c484ec2972418d782077bd3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/9af89a258442452687475e35e96c0743, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6f7970cf64444570a192a6abdf6d3f3f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b967f501065a4ea08636e0390fc93a0b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/32208904cbc745f8860dab4737b6b0ec, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/1b20fccf80474ff6be84695122d16556, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/7c3e90f41806471cbeeebb21f75898dc, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/f1ddbb18c3e447d795e7e4eeb74a5ddc, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/52b972123c1143c096582c79d2fb4e96, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8769462fd8e745b9a3704202c3ba384e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/49815c89cb4d47e188f1c01940df73c2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/4c775c2157074d26b8a34b0d5c7f1eb6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d6b66ce70d0d4b5fb4834b0652c596df, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/ca0917d8a49c409a983b80a1d56168b8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a58a75ad25da4f9288d764a6706913c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a3d8addfc5564e1db2cfff755da56413, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6c9e1e9a2a1a40638d2890257cfc39b6] to archive 2024-11-15T00:48:48,497 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:48:48,501 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/12ee2a5656d14d40ab286680006a3771 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/12ee2a5656d14d40ab286680006a3771 2024-11-15T00:48:48,503 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/18f7532207124f05a772ac1ba99fd38b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/18f7532207124f05a772ac1ba99fd38b 2024-11-15T00:48:48,504 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b57cd07500d84854bdd4d08ddca21ebd to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b57cd07500d84854bdd4d08ddca21ebd 2024-11-15T00:48:48,506 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/90f731425b384928a5f7aa7634d94d6e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/90f731425b384928a5f7aa7634d94d6e 2024-11-15T00:48:48,507 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8ec1d437e57847ebab95abdd7993369b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8ec1d437e57847ebab95abdd7993369b 2024-11-15T00:48:48,509 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d2446edcbe68451f972cadaf31191c66 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d2446edcbe68451f972cadaf31191c66 2024-11-15T00:48:48,510 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/09eb8a4a064c486baba88daf0cb86bbf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/09eb8a4a064c486baba88daf0cb86bbf 2024-11-15T00:48:48,511 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/36ade1a1f41a40258aaf2efb5b756d83 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/36ade1a1f41a40258aaf2efb5b756d83 2024-11-15T00:48:48,513 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/018070c79c484ec2972418d782077bd3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/018070c79c484ec2972418d782077bd3 2024-11-15T00:48:48,514 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/9af89a258442452687475e35e96c0743 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/9af89a258442452687475e35e96c0743 2024-11-15T00:48:48,515 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6f7970cf64444570a192a6abdf6d3f3f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6f7970cf64444570a192a6abdf6d3f3f 2024-11-15T00:48:48,516 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b967f501065a4ea08636e0390fc93a0b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/b967f501065a4ea08636e0390fc93a0b 2024-11-15T00:48:48,518 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/32208904cbc745f8860dab4737b6b0ec to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/32208904cbc745f8860dab4737b6b0ec 2024-11-15T00:48:48,519 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/1b20fccf80474ff6be84695122d16556 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/1b20fccf80474ff6be84695122d16556 2024-11-15T00:48:48,520 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/7c3e90f41806471cbeeebb21f75898dc to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/7c3e90f41806471cbeeebb21f75898dc 2024-11-15T00:48:48,522 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/f1ddbb18c3e447d795e7e4eeb74a5ddc to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/f1ddbb18c3e447d795e7e4eeb74a5ddc 2024-11-15T00:48:48,523 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/52b972123c1143c096582c79d2fb4e96 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/52b972123c1143c096582c79d2fb4e96 2024-11-15T00:48:48,524 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8769462fd8e745b9a3704202c3ba384e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/8769462fd8e745b9a3704202c3ba384e 2024-11-15T00:48:48,525 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/49815c89cb4d47e188f1c01940df73c2 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/49815c89cb4d47e188f1c01940df73c2 2024-11-15T00:48:48,526 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/4c775c2157074d26b8a34b0d5c7f1eb6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/4c775c2157074d26b8a34b0d5c7f1eb6 2024-11-15T00:48:48,527 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d6b66ce70d0d4b5fb4834b0652c596df to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/d6b66ce70d0d4b5fb4834b0652c596df 2024-11-15T00:48:48,528 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/ca0917d8a49c409a983b80a1d56168b8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/ca0917d8a49c409a983b80a1d56168b8 2024-11-15T00:48:48,529 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a58a75ad25da4f9288d764a6706913c0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a58a75ad25da4f9288d764a6706913c0 2024-11-15T00:48:48,530 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a3d8addfc5564e1db2cfff755da56413 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a3d8addfc5564e1db2cfff755da56413 2024-11-15T00:48:48,532 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6c9e1e9a2a1a40638d2890257cfc39b6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/6c9e1e9a2a1a40638d2890257cfc39b6 2024-11-15T00:48:48,543 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/537ecf1b00b74dae8d8dc88b5783604b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9eaa0ad88ea844c99a761b96ea715e6d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2ab47d7e90484e77986078d469533a4d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/6d3adb63b6664d54aad03a1edb358ebd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/883249b33c074f50b8ee08cf29af2a2c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/f899d544990f43a3acd3f8ebcc5dee4a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/e349a758f5104b90a03077da2ef0bbd3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/71643fb6f18e43bd968eb7f070d08112, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9a5d06ec68834c57af4f2e0526e9277b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2a6c1ec6d3104b6ebd4025c3086ac56c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/620a996a50524b7e92ec2c1362ba74b0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/fdc88f51de814ee2af46bbd046c144c4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/966c910f54324219a8fbc87ec0ba0d75, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/0a576c2f8d4749a7b1415653004341ac, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1099d2c1b9a446a0816afd89b0ee1087, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/ef95442ad70541118f6e237c000da1b7, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/bf84628124044ad195536daf4dcff845, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/c7139b26c12348c282798236d720ba77, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1976ccb79d264d9180f059c831fc0675, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/82cb531b5bc344c88b801b43b7b99d27, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/8aa5d5b1d6a749f3a61cb7f0a89027e0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/b80a6646063940f497c67b25b562b43f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/98f2c5452c274682b3e35446ad790912, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/d2630ff4134c4add91c150e5f7f48d26] to archive 2024-11-15T00:48:48,544 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:48:48,546 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/537ecf1b00b74dae8d8dc88b5783604b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/537ecf1b00b74dae8d8dc88b5783604b 2024-11-15T00:48:48,547 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9eaa0ad88ea844c99a761b96ea715e6d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9eaa0ad88ea844c99a761b96ea715e6d 2024-11-15T00:48:48,548 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2ab47d7e90484e77986078d469533a4d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2ab47d7e90484e77986078d469533a4d 2024-11-15T00:48:48,549 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/6d3adb63b6664d54aad03a1edb358ebd to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/6d3adb63b6664d54aad03a1edb358ebd 2024-11-15T00:48:48,551 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/883249b33c074f50b8ee08cf29af2a2c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/883249b33c074f50b8ee08cf29af2a2c 2024-11-15T00:48:48,552 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/f899d544990f43a3acd3f8ebcc5dee4a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/f899d544990f43a3acd3f8ebcc5dee4a 2024-11-15T00:48:48,554 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/e349a758f5104b90a03077da2ef0bbd3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/e349a758f5104b90a03077da2ef0bbd3 2024-11-15T00:48:48,555 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/71643fb6f18e43bd968eb7f070d08112 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/71643fb6f18e43bd968eb7f070d08112 2024-11-15T00:48:48,556 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9a5d06ec68834c57af4f2e0526e9277b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/9a5d06ec68834c57af4f2e0526e9277b 2024-11-15T00:48:48,557 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2a6c1ec6d3104b6ebd4025c3086ac56c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2a6c1ec6d3104b6ebd4025c3086ac56c 2024-11-15T00:48:48,559 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/620a996a50524b7e92ec2c1362ba74b0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/620a996a50524b7e92ec2c1362ba74b0 2024-11-15T00:48:48,560 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/fdc88f51de814ee2af46bbd046c144c4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/fdc88f51de814ee2af46bbd046c144c4 2024-11-15T00:48:48,561 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/966c910f54324219a8fbc87ec0ba0d75 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/966c910f54324219a8fbc87ec0ba0d75 2024-11-15T00:48:48,562 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/0a576c2f8d4749a7b1415653004341ac to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/0a576c2f8d4749a7b1415653004341ac 2024-11-15T00:48:48,563 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1099d2c1b9a446a0816afd89b0ee1087 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1099d2c1b9a446a0816afd89b0ee1087 2024-11-15T00:48:48,564 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/ef95442ad70541118f6e237c000da1b7 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/ef95442ad70541118f6e237c000da1b7 2024-11-15T00:48:48,566 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/bf84628124044ad195536daf4dcff845 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/bf84628124044ad195536daf4dcff845 2024-11-15T00:48:48,567 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/c7139b26c12348c282798236d720ba77 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/c7139b26c12348c282798236d720ba77 2024-11-15T00:48:48,568 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1976ccb79d264d9180f059c831fc0675 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/1976ccb79d264d9180f059c831fc0675 2024-11-15T00:48:48,569 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/82cb531b5bc344c88b801b43b7b99d27 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/82cb531b5bc344c88b801b43b7b99d27 2024-11-15T00:48:48,570 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/8aa5d5b1d6a749f3a61cb7f0a89027e0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/8aa5d5b1d6a749f3a61cb7f0a89027e0 2024-11-15T00:48:48,571 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/b80a6646063940f497c67b25b562b43f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/b80a6646063940f497c67b25b562b43f 2024-11-15T00:48:48,572 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/98f2c5452c274682b3e35446ad790912 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/98f2c5452c274682b3e35446ad790912 2024-11-15T00:48:48,574 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/d2630ff4134c4add91c150e5f7f48d26 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/d2630ff4134c4add91c150e5f7f48d26 2024-11-15T00:48:48,575 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2c4288d85b6444518b14e8c844ed8dd9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/33e3b9a1fc6c4a5599c1b635344bfc32, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/bdf81e80e8e047d5b51295dfb6f7a963, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c937bc877ea648949968fd9b212650bf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2e2060f659e34d2193e7644aac39269f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/60103e43a2eb411ca7eaf59b4df652eb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adf5c9116e744a03b78023424e917091, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adb8fb5817db47b289ba04a619248780, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c97cb1d8672546ef9dbdeb2209b72f40, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/66a249b74bbd42d3883d132171e00f31, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4e90f7d9b1644563b938d83c8ff6781d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5aaf5c32b340489f884c294f3888b094, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/7e5e56adffd44e87a842d7d35d78cb6c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/caf12d168bce464281c8f54b9f358a16, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/380fb3718f2c47df945be2d6e746ab01, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/188e586636da4f92bbc0af66ab7962b8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6856843c41804b188fd4c641b9935eb5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4ea022b61bad4d019f32c01b83d36c94, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/dc6f499212034273a61532fe42e700c6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/ee6322350da84474855fcc1cf8f4524f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6b68ea9bbff64b2c84f1b48574d8f261, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4a2374877ffe4bc1be40e03b1868cd54, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5b54c737ca3b40a5aececa040a1d7104] to archive 2024-11-15T00:48:48,577 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:48:48,578 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2c4288d85b6444518b14e8c844ed8dd9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2c4288d85b6444518b14e8c844ed8dd9 2024-11-15T00:48:48,580 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/33e3b9a1fc6c4a5599c1b635344bfc32 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/33e3b9a1fc6c4a5599c1b635344bfc32 2024-11-15T00:48:48,581 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/bdf81e80e8e047d5b51295dfb6f7a963 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/bdf81e80e8e047d5b51295dfb6f7a963 2024-11-15T00:48:48,583 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c937bc877ea648949968fd9b212650bf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c937bc877ea648949968fd9b212650bf 2024-11-15T00:48:48,584 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2e2060f659e34d2193e7644aac39269f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2e2060f659e34d2193e7644aac39269f 2024-11-15T00:48:48,585 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/60103e43a2eb411ca7eaf59b4df652eb to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/60103e43a2eb411ca7eaf59b4df652eb 2024-11-15T00:48:48,586 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adf5c9116e744a03b78023424e917091 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adf5c9116e744a03b78023424e917091 2024-11-15T00:48:48,587 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adb8fb5817db47b289ba04a619248780 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/adb8fb5817db47b289ba04a619248780 2024-11-15T00:48:48,588 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c97cb1d8672546ef9dbdeb2209b72f40 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c97cb1d8672546ef9dbdeb2209b72f40 2024-11-15T00:48:48,589 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/66a249b74bbd42d3883d132171e00f31 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/66a249b74bbd42d3883d132171e00f31 2024-11-15T00:48:48,590 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4e90f7d9b1644563b938d83c8ff6781d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4e90f7d9b1644563b938d83c8ff6781d 2024-11-15T00:48:48,591 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5aaf5c32b340489f884c294f3888b094 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5aaf5c32b340489f884c294f3888b094 2024-11-15T00:48:48,593 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/7e5e56adffd44e87a842d7d35d78cb6c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/7e5e56adffd44e87a842d7d35d78cb6c 2024-11-15T00:48:48,594 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/caf12d168bce464281c8f54b9f358a16 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/caf12d168bce464281c8f54b9f358a16 2024-11-15T00:48:48,595 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/380fb3718f2c47df945be2d6e746ab01 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/380fb3718f2c47df945be2d6e746ab01 2024-11-15T00:48:48,596 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/188e586636da4f92bbc0af66ab7962b8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/188e586636da4f92bbc0af66ab7962b8 2024-11-15T00:48:48,598 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6856843c41804b188fd4c641b9935eb5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6856843c41804b188fd4c641b9935eb5 2024-11-15T00:48:48,599 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4ea022b61bad4d019f32c01b83d36c94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4ea022b61bad4d019f32c01b83d36c94 2024-11-15T00:48:48,600 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/dc6f499212034273a61532fe42e700c6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/dc6f499212034273a61532fe42e700c6 2024-11-15T00:48:48,602 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/ee6322350da84474855fcc1cf8f4524f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/ee6322350da84474855fcc1cf8f4524f 2024-11-15T00:48:48,603 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6b68ea9bbff64b2c84f1b48574d8f261 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/6b68ea9bbff64b2c84f1b48574d8f261 2024-11-15T00:48:48,604 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4a2374877ffe4bc1be40e03b1868cd54 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/4a2374877ffe4bc1be40e03b1868cd54 2024-11-15T00:48:48,605 DEBUG [StoreCloser-TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5b54c737ca3b40a5aececa040a1d7104 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/5b54c737ca3b40a5aececa040a1d7104 2024-11-15T00:48:48,611 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/recovered.edits/416.seqid, newMaxSeqId=416, maxSeqId=1 2024-11-15T00:48:48,612 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. 2024-11-15T00:48:48,613 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] regionserver.HRegion(1676): Region close journal for 5b99d5a4d9ee135a84ccb828f4037b20: Waiting for close lock at 1731631727150Running coprocessor pre-close hooks at 1731631727150Disabling compacts and flushes for region at 1731631727150Disabling writes for close at 1731631727226 (+76 ms)Obtaining lock to block concurrent updates at 1731631727227 (+1 ms)Preparing flush snapshotting stores in 5b99d5a4d9ee135a84ccb828f4037b20 at 1731631727227Finished memstore snapshotting TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731631727227Flushing stores of TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20. at 1731631727227Flushing 5b99d5a4d9ee135a84ccb828f4037b20/A: creating writer at 1731631727227Flushing 5b99d5a4d9ee135a84ccb828f4037b20/A: appending metadata at 1731631727230 (+3 ms)Flushing 5b99d5a4d9ee135a84ccb828f4037b20/A: closing flushed file at 1731631727230Flushing 5b99d5a4d9ee135a84ccb828f4037b20/B: creating writer at 1731631727642 (+412 ms)Flushing 5b99d5a4d9ee135a84ccb828f4037b20/B: appending metadata at 1731631727645 (+3 ms)Flushing 5b99d5a4d9ee135a84ccb828f4037b20/B: closing flushed file at 1731631727645Flushing 5b99d5a4d9ee135a84ccb828f4037b20/C: creating writer at 1731631728061 (+416 ms)Flushing 5b99d5a4d9ee135a84ccb828f4037b20/C: appending metadata at 1731631728064 (+3 ms)Flushing 5b99d5a4d9ee135a84ccb828f4037b20/C: closing flushed file at 1731631728064Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60569ca8: reopening flushed file at 1731631728475 (+411 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@37a24b81: reopening flushed file at 1731631728481 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c1dc616: reopening flushed file at 1731631728486 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for 5b99d5a4d9ee135a84ccb828f4037b20 in 1267ms, sequenceid=413, compaction requested=true at 1731631728493 (+7 ms)Writing region close event to WAL at 1731631728607 (+114 ms)Running coprocessor post-close hooks at 1731631728612 (+5 ms)Closed at 1731631728612 2024-11-15T00:48:48,616 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=30}] handler.UnassignRegionHandler(157): Closed 5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:48,617 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=29 updating hbase:meta row=5b99d5a4d9ee135a84ccb828f4037b20, regionState=CLOSED 2024-11-15T00:48:48,619 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=30, ppid=29, state=RUNNABLE, hasLock=false; CloseRegionProcedure 5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:48:48,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-15T00:48:48,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseRegionProcedure 5b99d5a4d9ee135a84ccb828f4037b20, server=fb2f6cb7846a,42925,1731631698709 in 1.6320 sec 2024-11-15T00:48:48,623 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=29, resume processing ppid=28 2024-11-15T00:48:48,624 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, ppid=28, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=5b99d5a4d9ee135a84ccb828f4037b20, UNASSIGN in 1.6410 sec 2024-11-15T00:48:48,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-15T00:48:48,627 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6500 sec 2024-11-15T00:48:48,629 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631728629"}]},"ts":"1731631728629"} 2024-11-15T00:48:48,631 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:48:48,631 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:48:48,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6670 sec 2024-11-15T00:48:49,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-15T00:48:49,128 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:49,131 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:48:49,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:48:49,137 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=31, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:48:49,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T00:48:49,139 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=31, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:48:49,144 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:49,150 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/recovered.edits] 2024-11-15T00:48:49,155 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a0e691148e5546a2b6a9223b5b4642d8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/a0e691148e5546a2b6a9223b5b4642d8 2024-11-15T00:48:49,158 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/ab168a4442ed416ea32c1343b5388f12 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/ab168a4442ed416ea32c1343b5388f12 2024-11-15T00:48:49,160 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/fa4bb4c54f7e47cf84ab3ca34aedd173 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/fa4bb4c54f7e47cf84ab3ca34aedd173 2024-11-15T00:48:49,161 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/fd67c40d5a5f4c66861c76a2113cff2b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/A/fd67c40d5a5f4c66861c76a2113cff2b 2024-11-15T00:48:49,169 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/0c67bfb6ef5740fc98486adb90f63ed3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/0c67bfb6ef5740fc98486adb90f63ed3 2024-11-15T00:48:49,171 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/112e5ea7990949f6b84829b8f6511882 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/112e5ea7990949f6b84829b8f6511882 2024-11-15T00:48:49,177 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2e8b5a83cfaf4175bc18d71149219700 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/2e8b5a83cfaf4175bc18d71149219700 2024-11-15T00:48:49,185 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/7c7e4dfe55244f50967446e066e25baf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/B/7c7e4dfe55244f50967446e066e25baf 2024-11-15T00:48:49,191 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2a3531aec3584bdba260319d95cb8418 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/2a3531aec3584bdba260319d95cb8418 2024-11-15T00:48:49,192 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/3048a4c014f34ee1b89ee24afb662573 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/3048a4c014f34ee1b89ee24afb662573 2024-11-15T00:48:49,194 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/859db33a5b9f48868c23ffb211f1bbe3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/859db33a5b9f48868c23ffb211f1bbe3 2024-11-15T00:48:49,195 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c634fdc90e4d42a59baeecaa7e9763fb to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/c634fdc90e4d42a59baeecaa7e9763fb 2024-11-15T00:48:49,197 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/f88bcacba07e4a2fa57e3cb9b6004328 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/C/f88bcacba07e4a2fa57e3cb9b6004328 2024-11-15T00:48:49,201 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/recovered.edits/416.seqid to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20/recovered.edits/416.seqid 2024-11-15T00:48:49,201 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/5b99d5a4d9ee135a84ccb828f4037b20 2024-11-15T00:48:49,201 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:48:49,208 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=31, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:48:49,215 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-15T00:48:49,221 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:48:49,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T00:48:49,266 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:48:49,268 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=31, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:48:49,268 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:48:49,268 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631729268"}]},"ts":"9223372036854775807"} 2024-11-15T00:48:49,274 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:48:49,274 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 5b99d5a4d9ee135a84ccb828f4037b20, NAME => 'TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:48:49,274 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:48:49,274 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631729274"}]},"ts":"9223372036854775807"} 2024-11-15T00:48:49,277 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:48:49,278 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=31, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:48:49,280 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 146 msec 2024-11-15T00:48:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=31 2024-11-15T00:48:49,458 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:48:49,458 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:49,473 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=238 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1613740833_22 at /127.0.0.1:45208 [Waiting for operation #270] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1256560221_22 at /127.0.0.1:40202 [Waiting for operation #255] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;fb2f6cb7846a:42925-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=453 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=995 (was 838) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1980 (was 3111) 2024-11-15T00:48:49,480 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238, OpenFileDescriptor=453, MaxFileDescriptor=1048576, SystemLoadAverage=995, ProcessCount=11, AvailableMemoryMB=1977 2024-11-15T00:48:49,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:48:49,482 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:48:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:48:49,485 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:48:49,485 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:49,485 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 32 2024-11-15T00:48:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T00:48:49,486 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:48:49,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741922_1098 (size=959) 2024-11-15T00:48:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T00:48:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T00:48:49,902 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd 2024-11-15T00:48:49,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741923_1099 (size=53) 2024-11-15T00:48:50,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T00:48:50,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:48:50,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 88182fdac0358e5518c99ec7f861a42a, disabling compactions & flushes 2024-11-15T00:48:50,309 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:50,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:50,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. after waiting 0 ms 2024-11-15T00:48:50,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:50,309 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:50,309 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 88182fdac0358e5518c99ec7f861a42a: Waiting for close lock at 1731631730309Disabling compacts and flushes for region at 1731631730309Disabling writes for close at 1731631730309Writing region close event to WAL at 1731631730309Closed at 1731631730309 2024-11-15T00:48:50,311 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:48:50,312 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631730311"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631730311"}]},"ts":"1731631730311"} 2024-11-15T00:48:50,314 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:48:50,315 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:48:50,315 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631730315"}]},"ts":"1731631730315"} 2024-11-15T00:48:50,317 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:48:50,318 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, ASSIGN}] 2024-11-15T00:48:50,319 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, ASSIGN 2024-11-15T00:48:50,320 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,42925,1731631698709; forceNewPlan=false, retain=false 2024-11-15T00:48:50,471 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=88182fdac0358e5518c99ec7f861a42a, regionState=OPENING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:50,472 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=33, ppid=32, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, ASSIGN because future has completed 2024-11-15T00:48:50,473 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:48:50,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T00:48:50,628 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:50,629 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7752): Opening region: {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:48:50,629 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,629 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:48:50,629 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7794): checking encryption for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,629 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(7797): checking classloading for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,631 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,634 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88182fdac0358e5518c99ec7f861a42a columnFamilyName A 2024-11-15T00:48:50,634 DEBUG [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:50,635 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(327): Store=88182fdac0358e5518c99ec7f861a42a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:50,635 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,636 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88182fdac0358e5518c99ec7f861a42a columnFamilyName B 2024-11-15T00:48:50,636 DEBUG [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:50,637 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(327): Store=88182fdac0358e5518c99ec7f861a42a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:50,637 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,638 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88182fdac0358e5518c99ec7f861a42a columnFamilyName C 2024-11-15T00:48:50,638 DEBUG [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:50,638 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(327): Store=88182fdac0358e5518c99ec7f861a42a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:50,638 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1038): replaying wal for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,639 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,639 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,640 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1048): stopping wal replay for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,640 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1060): Cleaning up temporary data for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,641 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:48:50,642 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1093): writing seq id for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,644 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:48:50,645 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1114): Opened 88182fdac0358e5518c99ec7f861a42a; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72341724, jitterRate=0.07797569036483765}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:48:50,645 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:50,646 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegion(1006): Region open journal for 88182fdac0358e5518c99ec7f861a42a: Running coprocessor pre-open hook at 1731631730629Writing region info on filesystem at 1731631730629Initializing all the Stores at 1731631730630 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631730630Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631730631 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631730631Cleaning up temporary data from old regions at 1731631730640 (+9 ms)Running coprocessor post-open hooks at 1731631730645 (+5 ms)Region opened successfully at 1731631730646 (+1 ms) 2024-11-15T00:48:50,647 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., pid=34, masterSystemTime=1731631730625 2024-11-15T00:48:50,649 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:50,649 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=34}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:50,649 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=33 updating hbase:meta row=88182fdac0358e5518c99ec7f861a42a, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:50,651 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=34, ppid=33, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:48:50,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=34, resume processing ppid=33 2024-11-15T00:48:50,655 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, ppid=33, state=SUCCESS, hasLock=false; OpenRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 in 180 msec 2024-11-15T00:48:50,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=33, resume processing ppid=32 2024-11-15T00:48:50,657 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, ppid=32, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, ASSIGN in 337 msec 2024-11-15T00:48:50,657 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:48:50,658 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631730657"}]},"ts":"1731631730657"} 2024-11-15T00:48:50,659 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:48:50,660 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=32, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:48:50,661 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1780 sec 2024-11-15T00:48:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=32 2024-11-15T00:48:51,637 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:51,639 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594c9324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:51,639 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:51,639 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:51,641 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:51,641 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@463b223a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:51,642 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:51,644 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57956, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:51,644 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@655d0d49, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:51,645 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:51,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:51,647 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:51,648 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:48:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,46397,1731631697729 2024-11-15T00:48:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@2ed75f2b 2024-11-15T00:48:51,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:48:51,650 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:57970, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:48:51,654 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:48:51,655 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:48:51,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=35, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T00:48:51,673 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741924_1100 (size=995) 2024-11-15T00:48:52,074 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T00:48:52,075 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T00:48:52,078 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:48:52,086 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, REOPEN/MOVE}] 2024-11-15T00:48:52,087 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, REOPEN/MOVE 2024-11-15T00:48:52,088 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=88182fdac0358e5518c99ec7f861a42a, regionState=CLOSING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,090 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, REOPEN/MOVE because future has completed 2024-11-15T00:48:52,090 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:48:52,090 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:48:52,243 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(122): Close 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,243 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:48:52,243 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1722): Closing 88182fdac0358e5518c99ec7f861a42a, disabling compactions & flushes 2024-11-15T00:48:52,243 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:52,243 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:52,243 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. after waiting 0 ms 2024-11-15T00:48:52,243 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:52,248 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:48:52,249 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:52,249 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegion(1676): Region close journal for 88182fdac0358e5518c99ec7f861a42a: Waiting for close lock at 1731631732243Running coprocessor pre-close hooks at 1731631732243Disabling compacts and flushes for region at 1731631732243Disabling writes for close at 1731631732243Writing region close event to WAL at 1731631732244 (+1 ms)Running coprocessor post-close hooks at 1731631732249 (+5 ms)Closed at 1731631732249 2024-11-15T00:48:52,249 WARN [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] regionserver.HRegionServer(3299): Not adding moved region record: 88182fdac0358e5518c99ec7f861a42a to self. 2024-11-15T00:48:52,251 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=38}] handler.UnassignRegionHandler(157): Closed 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,252 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=88182fdac0358e5518c99ec7f861a42a, regionState=CLOSED 2024-11-15T00:48:52,254 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=38, ppid=37, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:48:52,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-15T00:48:52,257 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; CloseRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 in 165 msec 2024-11-15T00:48:52,258 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, REOPEN/MOVE; state=CLOSED, location=fb2f6cb7846a,42925,1731631698709; forceNewPlan=false, retain=true 2024-11-15T00:48:52,409 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=88182fdac0358e5518c99ec7f861a42a, regionState=OPENING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,411 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=37, ppid=36, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, REOPEN/MOVE because future has completed 2024-11-15T00:48:52,411 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:48:52,567 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:52,567 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7752): Opening region: {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:48:52,567 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,567 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:48:52,567 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7794): checking encryption for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,567 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(7797): checking classloading for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,571 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,577 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88182fdac0358e5518c99ec7f861a42a columnFamilyName A 2024-11-15T00:48:52,579 DEBUG [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:52,579 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(327): Store=88182fdac0358e5518c99ec7f861a42a/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:52,580 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,580 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88182fdac0358e5518c99ec7f861a42a columnFamilyName B 2024-11-15T00:48:52,581 DEBUG [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:52,581 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(327): Store=88182fdac0358e5518c99ec7f861a42a/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:52,581 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,582 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 88182fdac0358e5518c99ec7f861a42a columnFamilyName C 2024-11-15T00:48:52,582 DEBUG [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:52,583 INFO [StoreOpener-88182fdac0358e5518c99ec7f861a42a-1 {}] regionserver.HStore(327): Store=88182fdac0358e5518c99ec7f861a42a/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:48:52,583 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1038): replaying wal for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,584 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,585 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,586 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1048): stopping wal replay for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,586 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1060): Cleaning up temporary data for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,586 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:48:52,587 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1093): writing seq id for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,588 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1114): Opened 88182fdac0358e5518c99ec7f861a42a; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68060647, jitterRate=0.014182671904563904}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:48:52,588 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,589 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegion(1006): Region open journal for 88182fdac0358e5518c99ec7f861a42a: Running coprocessor pre-open hook at 1731631732567Writing region info on filesystem at 1731631732567Initializing all the Stores at 1731631732568 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631732568Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631732569 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631732569Cleaning up temporary data from old regions at 1731631732586 (+17 ms)Running coprocessor post-open hooks at 1731631732588 (+2 ms)Region opened successfully at 1731631732589 (+1 ms) 2024-11-15T00:48:52,590 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., pid=39, masterSystemTime=1731631732563 2024-11-15T00:48:52,592 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:52,592 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=39}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:52,593 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=37 updating hbase:meta row=88182fdac0358e5518c99ec7f861a42a, regionState=OPEN, openSeqNum=5, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,595 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=37, state=RUNNABLE, hasLock=false; OpenRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:48:52,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=37 2024-11-15T00:48:52,597 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=37, state=SUCCESS, hasLock=false; OpenRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 in 184 msec 2024-11-15T00:48:52,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=37, resume processing ppid=36 2024-11-15T00:48:52,602 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, ppid=36, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, REOPEN/MOVE in 513 msec 2024-11-15T00:48:52,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-15T00:48:52,606 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 525 msec 2024-11-15T00:48:52,611 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 951 msec 2024-11-15T00:48:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=35 2024-11-15T00:48:52,613 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:52,614 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcb20d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,614 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:52,614 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:52,615 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:52,615 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7df99fb7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,616 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:52,619 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:52,625 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45422, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:52,625 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd7ed2a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,628 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e853879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,628 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:52,629 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:52,629 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:52,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ac20d15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:52,630 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:52,631 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45436, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:52,631 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5dc70b1c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,633 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a00ea56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,633 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:52,633 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:52,636 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3329bdb5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:52,637 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:52,638 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:52,639 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45454, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:52,639 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@681be1c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,641 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636d090d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,641 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:52,641 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:52,642 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3b00af65, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:52,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:52,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:52,645 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45474, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:52,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@291d2a6a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,646 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0133ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,646 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:52,646 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:52,647 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:52,647 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ed2cd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:52,648 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:52,650 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45500, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:52,651 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c9236f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc5db83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,652 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:52,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:52,654 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:52,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:52,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:52,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c9973d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,654 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:52,655 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:52,656 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45510, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:52,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@680e861f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,660 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4870d163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,660 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:52,660 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:52,661 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bb3bcd4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:52,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:52,662 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:52,663 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45526, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:52,664 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70ac8dce, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,671 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53e9634b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,671 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:52,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:52,673 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:52,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:52,673 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f6f899f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:52,674 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:52,676 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45548, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:52,677 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e5fc009, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,677 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f7e914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,678 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:48:52,678 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:48:52,681 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:48:52,681 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:48:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:48:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a4102d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:48:52,682 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:48:52,683 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:48:52,686 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:45566, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:48:52,687 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@30213712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:48:52,688 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:52,688 DEBUG [Thread-532 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:52,688 DEBUG [Thread-534 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:52,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees 2024-11-15T00:48:52,689 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:52,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:48:52,691 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35586, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:52,692 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35596, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:52,692 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:52,693 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=40, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:52,693 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=40, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:52,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:48:52,697 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:48:52,701 DEBUG [Thread-536 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:52,702 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:52,704 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35612, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:52,707 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:48:52,712 DEBUG [Thread-539 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:52,714 DEBUG [Thread-528 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:52,714 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:52,715 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:52,716 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:52,717 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35628, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:52,718 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:48:52,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:48:52,722 DEBUG [Thread-530 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:52,723 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:52,724 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:52,726 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:48:52,736 DEBUG [Thread-543 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:52,736 DEBUG [Thread-541 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:52,737 DEBUG [Thread-545 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:48:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:48:52,737 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:52,738 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:48:52,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35638, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:52,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:52,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35650, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:48:52,741 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:48:52,741 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:48:52,743 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:48:52,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,765 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:52,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:48:52,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:48:52,846 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:48:52,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:52,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:52,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:52,847 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:52,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:52,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:52,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:52,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631792848, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,864 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111513d4efb5008f43c1aa5f2cd916af804c_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631732744/Put/seqid=0 2024-11-15T00:48:52,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:52,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631792849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,864 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:52,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631792853, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,866 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:52,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631792858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,867 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:52,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631792861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:52,870 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,871 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:52,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741925_1101 (size=17034) 2024-11-15T00:48:52,925 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:52,935 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111513d4efb5008f43c1aa5f2cd916af804c_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111513d4efb5008f43c1aa5f2cd916af804c_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:52,939 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/7b42f1ee8d884ef9ac3f2982699a89c0, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:52,955 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/7b42f1ee8d884ef9ac3f2982699a89c0 is 175, key is test_row_0/A:col10/1731631732744/Put/seqid=0 2024-11-15T00:48:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:52,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631792978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,983 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:52,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631792978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,984 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,985 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:52,986 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:52,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631792982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,987 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:52,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:52,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631792982, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:52,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:52,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631792983, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:52,996 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,000 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:48:53,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:53,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,001 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:48:53,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741926_1102 (size=48139) 2024-11-15T00:48:53,018 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/7b42f1ee8d884ef9ac3f2982699a89c0 2024-11-15T00:48:53,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/c0d9b8eaba614c3bab7dfb28526db7f5 is 50, key is test_row_0/B:col10/1731631732744/Put/seqid=0 2024-11-15T00:48:53,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741927_1103 (size=12001) 2024-11-15T00:48:53,104 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/c0d9b8eaba614c3bab7dfb28526db7f5 2024-11-15T00:48:53,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:48:53,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:53,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,160 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,183 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/0f9cbb908d5641f3a3a02cc057a4ab64 is 50, key is test_row_0/C:col10/1731631732744/Put/seqid=0 2024-11-15T00:48:53,195 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:53,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631793190, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:53,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,196 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741928_1104 (size=12001) 2024-11-15T00:48:53,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631793191, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,198 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,198 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:53,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631793197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,199 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:53,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631793208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,212 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,213 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:53,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631793211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,214 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:48:53,317 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:48:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:53,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,318 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,318 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:48:53,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:53,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,472 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,508 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:53,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631793507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:53,508 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:53,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631793507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,509 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631793507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:53,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631793516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:53,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,519 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,520 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,527 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:53,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631793527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:53,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:53,595 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/0f9cbb908d5641f3a3a02cc057a4ab64 2024-11-15T00:48:53,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/7b42f1ee8d884ef9ac3f2982699a89c0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/7b42f1ee8d884ef9ac3f2982699a89c0 2024-11-15T00:48:53,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:48:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,626 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] handler.RSProcedureHandler(58): pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=41 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=41 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:53,634 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/7b42f1ee8d884ef9ac3f2982699a89c0, entries=250, sequenceid=16, filesize=47.0 K 2024-11-15T00:48:53,643 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/c0d9b8eaba614c3bab7dfb28526db7f5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/c0d9b8eaba614c3bab7dfb28526db7f5 2024-11-15T00:48:53,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/c0d9b8eaba614c3bab7dfb28526db7f5, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T00:48:53,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/0f9cbb908d5641f3a3a02cc057a4ab64 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/0f9cbb908d5641f3a3a02cc057a4ab64 2024-11-15T00:48:53,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/0f9cbb908d5641f3a3a02cc057a4ab64, entries=150, sequenceid=16, filesize=11.7 K 2024-11-15T00:48:53,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 88182fdac0358e5518c99ec7f861a42a in 913ms, sequenceid=16, compaction requested=false 2024-11-15T00:48:53,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:53,781 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=41 2024-11-15T00:48:53,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:53,781 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T00:48:53,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111517326d5c7e1b4a26ac6c6413fcfac1af_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631732854/Put/seqid=0 2024-11-15T00:48:53,821 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741929_1105 (size=12154) 2024-11-15T00:48:53,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:48:53,832 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111517326d5c7e1b4a26ac6c6413fcfac1af_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111517326d5c7e1b4a26ac6c6413fcfac1af_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:53,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/b103ea7028ad4f8faa9bc1e046a3cc45, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:53,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/b103ea7028ad4f8faa9bc1e046a3cc45 is 175, key is test_row_0/A:col10/1731631732854/Put/seqid=0 2024-11-15T00:48:53,844 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741930_1106 (size=30955) 2024-11-15T00:48:53,848 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/b103ea7028ad4f8faa9bc1e046a3cc45 2024-11-15T00:48:53,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/db409ca7a39844d38aeb774ca156113c is 50, key is test_row_0/B:col10/1731631732854/Put/seqid=0 2024-11-15T00:48:53,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741931_1107 (size=12001) 2024-11-15T00:48:53,892 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/db409ca7a39844d38aeb774ca156113c 2024-11-15T00:48:53,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/463fee77addc40bc93d42e4f27d9a742 is 50, key is test_row_0/C:col10/1731631732854/Put/seqid=0 2024-11-15T00:48:53,926 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741932_1108 (size=12001) 2024-11-15T00:48:54,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:54,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:54,040 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631794037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631794037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,041 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631794037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631794040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,042 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631794040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631794147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631794148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,149 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,149 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631794148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,150 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631794148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,151 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,152 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,162 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:48:54,331 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/463fee77addc40bc93d42e4f27d9a742 2024-11-15T00:48:54,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/b103ea7028ad4f8faa9bc1e046a3cc45 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b103ea7028ad4f8faa9bc1e046a3cc45 2024-11-15T00:48:54,345 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b103ea7028ad4f8faa9bc1e046a3cc45, entries=150, sequenceid=40, filesize=30.2 K 2024-11-15T00:48:54,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/db409ca7a39844d38aeb774ca156113c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/db409ca7a39844d38aeb774ca156113c 2024-11-15T00:48:54,361 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/db409ca7a39844d38aeb774ca156113c, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T00:48:54,361 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631794357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,362 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631794358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,362 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631794358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/463fee77addc40bc93d42e4f27d9a742 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/463fee77addc40bc93d42e4f27d9a742 2024-11-15T00:48:54,363 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631794358, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,363 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,364 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,371 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/463fee77addc40bc93d42e4f27d9a742, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T00:48:54,372 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=67.09 KB/68700 for 88182fdac0358e5518c99ec7f861a42a in 591ms, sequenceid=40, compaction requested=false 2024-11-15T00:48:54,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:54,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:54,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=41}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=41 2024-11-15T00:48:54,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=41 2024-11-15T00:48:54,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=40 2024-11-15T00:48:54,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6810 sec 2024-11-15T00:48:54,377 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=40, table=TestAcidGuarantees in 1.6880 sec 2024-11-15T00:48:54,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:54,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T00:48:54,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a2a337351d9942e7a4981991d4125c91_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631734034/Put/seqid=0 2024-11-15T00:48:54,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741933_1109 (size=17034) 2024-11-15T00:48:54,711 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:54,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631794706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,712 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631794708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,712 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,713 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,715 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631794710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,716 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631794712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,722 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a2a337351d9942e7a4981991d4125c91_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a2a337351d9942e7a4981991d4125c91_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,723 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/b8636236d07841a2aa3e8ea5f4b07cba, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:54,724 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/b8636236d07841a2aa3e8ea5f4b07cba is 175, key is test_row_0/A:col10/1731631734034/Put/seqid=0 2024-11-15T00:48:54,756 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741934_1110 (size=48139) 2024-11-15T00:48:54,758 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=55, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/b8636236d07841a2aa3e8ea5f4b07cba 2024-11-15T00:48:54,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/8501394c900e4f8fac136a8693ddbf7e is 50, key is test_row_0/B:col10/1731631734034/Put/seqid=0 2024-11-15T00:48:54,795 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741935_1111 (size=12001) 2024-11-15T00:48:54,798 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/8501394c900e4f8fac136a8693ddbf7e 2024-11-15T00:48:54,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/6066bd86a95048afb125c147fa644eaf is 50, key is test_row_0/C:col10/1731631734034/Put/seqid=0 2024-11-15T00:48:54,823 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631794817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631794817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,824 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,825 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741936_1112 (size=12001) 2024-11-15T00:48:54,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631794830, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,833 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631794831, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:54,833 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=40 2024-11-15T00:48:54,837 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:54,838 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-15T00:48:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:48:54,840 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:54,841 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:54,841 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:54,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:48:54,994 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T00:48:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:54,994 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:54,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:54,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:55,031 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631795027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631795037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631795037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,045 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631795037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,049 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,049 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631795048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,050 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 88182fdac0358e5518c99ec7f861a42a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,050 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T00:48:55,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:55,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:55,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:55,147 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:55,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:55,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:55,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:48:55,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=55 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/6066bd86a95048afb125c147fa644eaf 2024-11-15T00:48:55,257 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/b8636236d07841a2aa3e8ea5f4b07cba as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b8636236d07841a2aa3e8ea5f4b07cba 2024-11-15T00:48:55,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b8636236d07841a2aa3e8ea5f4b07cba, entries=250, sequenceid=55, filesize=47.0 K 2024-11-15T00:48:55,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/8501394c900e4f8fac136a8693ddbf7e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8501394c900e4f8fac136a8693ddbf7e 2024-11-15T00:48:55,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8501394c900e4f8fac136a8693ddbf7e, entries=150, sequenceid=55, filesize=11.7 K 2024-11-15T00:48:55,286 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/6066bd86a95048afb125c147fa644eaf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6066bd86a95048afb125c147fa644eaf 2024-11-15T00:48:55,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6066bd86a95048afb125c147fa644eaf, entries=150, sequenceid=55, filesize=11.7 K 2024-11-15T00:48:55,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 88182fdac0358e5518c99ec7f861a42a in 625ms, sequenceid=55, compaction requested=true 2024-11-15T00:48:55,294 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:55,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:55,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:55,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:55,294 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:55,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:55,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:55,294 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:48:55,294 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:55,300 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:55,300 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/B is initiating minor compaction (all files) 2024-11-15T00:48:55,300 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/B in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:55,300 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-15T00:48:55,300 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/c0d9b8eaba614c3bab7dfb28526db7f5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/db409ca7a39844d38aeb774ca156113c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8501394c900e4f8fac136a8693ddbf7e] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=35.2 K 2024-11-15T00:48:55,301 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:55,301 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T00:48:55,301 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 127233 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:55,301 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/A is initiating minor compaction (all files) 2024-11-15T00:48:55,301 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/A in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:55,301 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/7b42f1ee8d884ef9ac3f2982699a89c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b103ea7028ad4f8faa9bc1e046a3cc45, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b8636236d07841a2aa3e8ea5f4b07cba] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=124.3 K 2024-11-15T00:48:55,301 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:55,302 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/7b42f1ee8d884ef9ac3f2982699a89c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b103ea7028ad4f8faa9bc1e046a3cc45, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b8636236d07841a2aa3e8ea5f4b07cba] 2024-11-15T00:48:55,303 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c0d9b8eaba614c3bab7dfb28526db7f5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631732744 2024-11-15T00:48:55,303 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b42f1ee8d884ef9ac3f2982699a89c0, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631732744 2024-11-15T00:48:55,303 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting db409ca7a39844d38aeb774ca156113c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731631732844 2024-11-15T00:48:55,304 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b103ea7028ad4f8faa9bc1e046a3cc45, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731631732844 2024-11-15T00:48:55,304 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8636236d07841a2aa3e8ea5f4b07cba, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731631734034 2024-11-15T00:48:55,305 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8501394c900e4f8fac136a8693ddbf7e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731631734034 2024-11-15T00:48:55,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111514dd7d00142344a8b4dc9a3ee1534eb2_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631734704/Put/seqid=0 2024-11-15T00:48:55,341 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:55,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:55,345 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:55,347 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:55,347 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/5a8501dd3b1f4094af2e31d1cac773c4 is 50, key is test_row_0/B:col10/1731631734034/Put/seqid=0 2024-11-15T00:48:55,357 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115fb2c38e08c1f467e94b4b83b26caf633_88182fdac0358e5518c99ec7f861a42a store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:55,364 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115fb2c38e08c1f467e94b4b83b26caf633_88182fdac0358e5518c99ec7f861a42a, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:55,364 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115fb2c38e08c1f467e94b4b83b26caf633_88182fdac0358e5518c99ec7f861a42a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:55,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741937_1113 (size=12154) 2024-11-15T00:48:55,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741938_1114 (size=12104) 2024-11-15T00:48:55,386 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/5a8501dd3b1f4094af2e31d1cac773c4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/5a8501dd3b1f4094af2e31d1cac773c4 2024-11-15T00:48:55,389 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631795383, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,390 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,394 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/B of 88182fdac0358e5518c99ec7f861a42a into 5a8501dd3b1f4094af2e31d1cac773c4(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:55,394 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:55,394 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/B, priority=13, startTime=1731631735294; duration=0sec 2024-11-15T00:48:55,394 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:55,394 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:B 2024-11-15T00:48:55,394 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:55,397 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:55,397 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/C is initiating minor compaction (all files) 2024-11-15T00:48:55,397 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/C in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:55,397 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/0f9cbb908d5641f3a3a02cc057a4ab64, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/463fee77addc40bc93d42e4f27d9a742, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6066bd86a95048afb125c147fa644eaf] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=35.2 K 2024-11-15T00:48:55,398 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f9cbb908d5641f3a3a02cc057a4ab64, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631732744 2024-11-15T00:48:55,398 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 463fee77addc40bc93d42e4f27d9a742, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731631732844 2024-11-15T00:48:55,399 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6066bd86a95048afb125c147fa644eaf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731631734034 2024-11-15T00:48:55,399 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631795387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631795387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,400 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,400 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631795389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,401 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741939_1115 (size=4469) 2024-11-15T00:48:55,422 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#A#compaction#95 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:55,422 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/10d0c988a7c2481fb2adbec429eeab6f is 175, key is test_row_0/A:col10/1731631734034/Put/seqid=0 2024-11-15T00:48:55,435 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#C#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:55,435 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/78c754d421854bb6bd31e59c5141bb17 is 50, key is test_row_0/C:col10/1731631734034/Put/seqid=0 2024-11-15T00:48:55,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:48:55,475 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741940_1116 (size=31058) 2024-11-15T00:48:55,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741941_1117 (size=12104) 2024-11-15T00:48:55,483 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/10d0c988a7c2481fb2adbec429eeab6f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/10d0c988a7c2481fb2adbec429eeab6f 2024-11-15T00:48:55,492 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/A of 88182fdac0358e5518c99ec7f861a42a into 10d0c988a7c2481fb2adbec429eeab6f(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:55,492 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:55,493 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/A, priority=13, startTime=1731631735294; duration=0sec 2024-11-15T00:48:55,493 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:55,493 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:A 2024-11-15T00:48:55,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631795497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,500 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,512 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631795507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,513 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631795507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,513 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631795508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,514 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,709 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631795707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,710 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,721 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,721 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631795716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631795716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,722 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:55,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631795726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:55,737 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:55,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:55,793 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111514dd7d00142344a8b4dc9a3ee1534eb2_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111514dd7d00142344a8b4dc9a3ee1534eb2_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/20b54b1fb03b4573a625e96f0ca4d7ea, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:55,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/20b54b1fb03b4573a625e96f0ca4d7ea is 175, key is test_row_0/A:col10/1731631734704/Put/seqid=0 2024-11-15T00:48:55,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741942_1118 (size=30955) 2024-11-15T00:48:55,799 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=76, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/20b54b1fb03b4573a625e96f0ca4d7ea 2024-11-15T00:48:55,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/e2888002e68f4caea4d5164886c37546 is 50, key is test_row_0/B:col10/1731631734704/Put/seqid=0 2024-11-15T00:48:55,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741943_1119 (size=12001) 2024-11-15T00:48:55,889 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/78c754d421854bb6bd31e59c5141bb17 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/78c754d421854bb6bd31e59c5141bb17 2024-11-15T00:48:55,898 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/C of 88182fdac0358e5518c99ec7f861a42a into 78c754d421854bb6bd31e59c5141bb17(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:55,898 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:55,898 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/C, priority=13, startTime=1731631735294; duration=0sec 2024-11-15T00:48:55,898 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:55,898 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:C 2024-11-15T00:48:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:48:56,022 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:56,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631796016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:56,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631796027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:56,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631796027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,028 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:56,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:56,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631796047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,050 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:56,231 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/e2888002e68f4caea4d5164886c37546 2024-11-15T00:48:56,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/87ab52a69262448691b87d37ac9f651f is 50, key is test_row_0/C:col10/1731631734704/Put/seqid=0 2024-11-15T00:48:56,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741944_1120 (size=12001) 2024-11-15T00:48:56,528 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:56,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631796528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:56,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:56,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631796538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:56,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:56,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631796540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:56,564 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:56,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631796561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:56,566 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:56,659 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/87ab52a69262448691b87d37ac9f651f 2024-11-15T00:48:56,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/20b54b1fb03b4573a625e96f0ca4d7ea as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/20b54b1fb03b4573a625e96f0ca4d7ea 2024-11-15T00:48:56,679 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/20b54b1fb03b4573a625e96f0ca4d7ea, entries=150, sequenceid=76, filesize=30.2 K 2024-11-15T00:48:56,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/e2888002e68f4caea4d5164886c37546 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e2888002e68f4caea4d5164886c37546 2024-11-15T00:48:56,702 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e2888002e68f4caea4d5164886c37546, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T00:48:56,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/87ab52a69262448691b87d37ac9f651f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/87ab52a69262448691b87d37ac9f651f 2024-11-15T00:48:56,719 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/87ab52a69262448691b87d37ac9f651f, entries=150, sequenceid=76, filesize=11.7 K 2024-11-15T00:48:56,720 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 88182fdac0358e5518c99ec7f861a42a in 1419ms, sequenceid=76, compaction requested=false 2024-11-15T00:48:56,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:56,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:56,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-15T00:48:56,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-15T00:48:56,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-15T00:48:56,724 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8810 sec 2024-11-15T00:48:56,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.8860 sec 2024-11-15T00:48:56,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-15T00:48:56,987 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:56,988 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:56,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-15T00:48:56,991 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:56,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:48:56,992 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:56,993 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:57,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:57,071 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:57,072 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T00:48:57,084 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157c48b0dd29f042209a74382c19f2e9af_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631735383/Put/seqid=0 2024-11-15T00:48:57,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:48:57,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741945_1121 (size=12154) 2024-11-15T00:48:57,110 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:57,119 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157c48b0dd29f042209a74382c19f2e9af_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c48b0dd29f042209a74382c19f2e9af_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:57,123 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/dd45d546a93444338a3bf6f93184bb1a, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:57,124 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/dd45d546a93444338a3bf6f93184bb1a is 175, key is test_row_0/A:col10/1731631735383/Put/seqid=0 2024-11-15T00:48:57,147 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T00:48:57,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:57,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,148 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,148 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741946_1122 (size=30955) 2024-11-15T00:48:57,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:57,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631797179, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,185 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:57,301 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:57,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631797299, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:57,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T00:48:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:57,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:57,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:48:57,461 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T00:48:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,461 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:57,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631797507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:57,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:57,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631797539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:57,542 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 88182fdac0358e5518c99ec7f861a42a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2159 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:57,553 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:57,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631797546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:57,554 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 88182fdac0358e5518c99ec7f861a42a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:57,556 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/dd45d546a93444338a3bf6f93184bb1a 2024-11-15T00:48:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,556 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:57,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:57,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631797557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:57,560 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 88182fdac0358e5518c99ec7f861a42a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2173 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,562 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:57,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/fe64fed092994467b8e4d5e076808365 is 50, key is test_row_0/B:col10/1731631735383/Put/seqid=0 2024-11-15T00:48:57,580 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:57,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631797578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:57,581 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741947_1123 (size=12001) 2024-11-15T00:48:57,581 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 88182fdac0358e5518c99ec7f861a42a of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2192 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:57,581 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,582 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:57,584 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/fe64fed092994467b8e4d5e076808365 2024-11-15T00:48:57,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/25e00f2d466d4fbebad49d0882d54cc1 is 50, key is test_row_0/C:col10/1731631735383/Put/seqid=0 2024-11-15T00:48:57,615 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T00:48:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:48:57,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:48:57,638 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741948_1124 (size=12001) 2024-11-15T00:48:57,638 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/25e00f2d466d4fbebad49d0882d54cc1 2024-11-15T00:48:57,644 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/dd45d546a93444338a3bf6f93184bb1a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/dd45d546a93444338a3bf6f93184bb1a 2024-11-15T00:48:57,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/dd45d546a93444338a3bf6f93184bb1a, entries=150, sequenceid=95, filesize=30.2 K 2024-11-15T00:48:57,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/fe64fed092994467b8e4d5e076808365 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/fe64fed092994467b8e4d5e076808365 2024-11-15T00:48:57,674 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/fe64fed092994467b8e4d5e076808365, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T00:48:57,675 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/25e00f2d466d4fbebad49d0882d54cc1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/25e00f2d466d4fbebad49d0882d54cc1 2024-11-15T00:48:57,684 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/25e00f2d466d4fbebad49d0882d54cc1, entries=150, sequenceid=95, filesize=11.7 K 2024-11-15T00:48:57,685 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 88182fdac0358e5518c99ec7f861a42a in 614ms, sequenceid=95, compaction requested=true 2024-11-15T00:48:57,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:57,685 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:57,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:57,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:57,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:57,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:57,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:57,686 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:48:57,687 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:57,687 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/A is initiating minor compaction (all files) 2024-11-15T00:48:57,687 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/A in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,687 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/10d0c988a7c2481fb2adbec429eeab6f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/20b54b1fb03b4573a625e96f0ca4d7ea, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/dd45d546a93444338a3bf6f93184bb1a] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=90.8 K 2024-11-15T00:48:57,688 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:57,688 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,688 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/10d0c988a7c2481fb2adbec429eeab6f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/20b54b1fb03b4573a625e96f0ca4d7ea, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/dd45d546a93444338a3bf6f93184bb1a] 2024-11-15T00:48:57,690 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10d0c988a7c2481fb2adbec429eeab6f, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731631734034 2024-11-15T00:48:57,693 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20b54b1fb03b4573a625e96f0ca4d7ea, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731631734704 2024-11-15T00:48:57,694 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:57,694 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/B is initiating minor compaction (all files) 2024-11-15T00:48:57,694 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/B in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,694 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/5a8501dd3b1f4094af2e31d1cac773c4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e2888002e68f4caea4d5164886c37546, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/fe64fed092994467b8e4d5e076808365] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=35.3 K 2024-11-15T00:48:57,695 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd45d546a93444338a3bf6f93184bb1a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631735383 2024-11-15T00:48:57,695 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a8501dd3b1f4094af2e31d1cac773c4, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731631734034 2024-11-15T00:48:57,697 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e2888002e68f4caea4d5164886c37546, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731631734704 2024-11-15T00:48:57,697 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting fe64fed092994467b8e4d5e076808365, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631735383 2024-11-15T00:48:57,716 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:57,718 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#B#compaction#104 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:57,719 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/8431745a633541bd8a19dfa9654135a2 is 50, key is test_row_0/B:col10/1731631735383/Put/seqid=0 2024-11-15T00:48:57,725 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115db00b7e1b18242fa829306e2384a3a77_88182fdac0358e5518c99ec7f861a42a store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:57,729 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115db00b7e1b18242fa829306e2384a3a77_88182fdac0358e5518c99ec7f861a42a, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:57,729 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115db00b7e1b18242fa829306e2384a3a77_88182fdac0358e5518c99ec7f861a42a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:57,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741950_1126 (size=4469) 2024-11-15T00:48:57,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741949_1125 (size=12207) 2024-11-15T00:48:57,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-15T00:48:57,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:57,773 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T00:48:57,804 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155e2e9ac82fef41bda3c64b5cc333e81f_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631737176/Put/seqid=0 2024-11-15T00:48:57,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741951_1127 (size=12154) 2024-11-15T00:48:57,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:57,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:57,821 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:57,827 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155e2e9ac82fef41bda3c64b5cc333e81f_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155e2e9ac82fef41bda3c64b5cc333e81f_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:57,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ccf3a87dab0f4f3d8825ffc18c3cb30b, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:57,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ccf3a87dab0f4f3d8825ffc18c3cb30b is 175, key is test_row_0/A:col10/1731631737176/Put/seqid=0 2024-11-15T00:48:57,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741952_1128 (size=30955) 2024-11-15T00:48:57,905 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:57,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631797904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:57,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:58,017 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:58,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631798017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:58,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:48:58,150 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#A#compaction#103 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:58,151 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/54cbdc6fe7f449038158df06fdee81f4 is 175, key is test_row_0/A:col10/1731631735383/Put/seqid=0 2024-11-15T00:48:58,156 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/8431745a633541bd8a19dfa9654135a2 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8431745a633541bd8a19dfa9654135a2 2024-11-15T00:48:58,164 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/B of 88182fdac0358e5518c99ec7f861a42a into 8431745a633541bd8a19dfa9654135a2(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:58,165 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:58,165 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/B, priority=13, startTime=1731631737686; duration=0sec 2024-11-15T00:48:58,165 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:58,165 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:B 2024-11-15T00:48:58,165 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:58,166 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:58,166 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/C is initiating minor compaction (all files) 2024-11-15T00:48:58,166 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/C in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:58,166 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/78c754d421854bb6bd31e59c5141bb17, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/87ab52a69262448691b87d37ac9f651f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/25e00f2d466d4fbebad49d0882d54cc1] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=35.3 K 2024-11-15T00:48:58,167 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 78c754d421854bb6bd31e59c5141bb17, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=55, earliestPutTs=1731631734034 2024-11-15T00:48:58,167 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 87ab52a69262448691b87d37ac9f651f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1731631734704 2024-11-15T00:48:58,168 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 25e00f2d466d4fbebad49d0882d54cc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631735383 2024-11-15T00:48:58,168 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741953_1129 (size=31161) 2024-11-15T00:48:58,178 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#C#compaction#106 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:58,178 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/cdfd346e9ad349d4a82293680a1305b0 is 50, key is test_row_0/C:col10/1731631735383/Put/seqid=0 2024-11-15T00:48:58,191 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741954_1130 (size=12207) 2024-11-15T00:48:58,197 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/cdfd346e9ad349d4a82293680a1305b0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/cdfd346e9ad349d4a82293680a1305b0 2024-11-15T00:48:58,203 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/C of 88182fdac0358e5518c99ec7f861a42a into cdfd346e9ad349d4a82293680a1305b0(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:58,203 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:58,203 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/C, priority=13, startTime=1731631737686; duration=0sec 2024-11-15T00:48:58,203 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:58,203 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:C 2024-11-15T00:48:58,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:58,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631798227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:58,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:58,241 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=115, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ccf3a87dab0f4f3d8825ffc18c3cb30b 2024-11-15T00:48:58,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/9d892d81f1bc4f9aac55e9fdc1814651 is 50, key is test_row_0/B:col10/1731631737176/Put/seqid=0 2024-11-15T00:48:58,263 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741955_1131 (size=12001) 2024-11-15T00:48:58,265 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/9d892d81f1bc4f9aac55e9fdc1814651 2024-11-15T00:48:58,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/3d63a125949f4ddda74a3b7f2d4b62e3 is 50, key is test_row_0/C:col10/1731631737176/Put/seqid=0 2024-11-15T00:48:58,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741956_1132 (size=12001) 2024-11-15T00:48:58,287 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=115 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/3d63a125949f4ddda74a3b7f2d4b62e3 2024-11-15T00:48:58,296 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ccf3a87dab0f4f3d8825ffc18c3cb30b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ccf3a87dab0f4f3d8825ffc18c3cb30b 2024-11-15T00:48:58,304 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ccf3a87dab0f4f3d8825ffc18c3cb30b, entries=150, sequenceid=115, filesize=30.2 K 2024-11-15T00:48:58,305 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/9d892d81f1bc4f9aac55e9fdc1814651 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9d892d81f1bc4f9aac55e9fdc1814651 2024-11-15T00:48:58,311 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9d892d81f1bc4f9aac55e9fdc1814651, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T00:48:58,313 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/3d63a125949f4ddda74a3b7f2d4b62e3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/3d63a125949f4ddda74a3b7f2d4b62e3 2024-11-15T00:48:58,321 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/3d63a125949f4ddda74a3b7f2d4b62e3, entries=150, sequenceid=115, filesize=11.7 K 2024-11-15T00:48:58,322 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 88182fdac0358e5518c99ec7f861a42a in 549ms, sequenceid=115, compaction requested=false 2024-11-15T00:48:58,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:58,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:58,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-15T00:48:58,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-15T00:48:58,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-15T00:48:58,327 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3310 sec 2024-11-15T00:48:58,333 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.3420 sec 2024-11-15T00:48:58,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:58,540 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:48:58,540 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T00:48:58,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d9e2acfbbd441b0a17cc2d97554d5fa_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631737900/Put/seqid=0 2024-11-15T00:48:58,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741957_1133 (size=14694) 2024-11-15T00:48:58,578 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/54cbdc6fe7f449038158df06fdee81f4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/54cbdc6fe7f449038158df06fdee81f4 2024-11-15T00:48:58,585 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/A of 88182fdac0358e5518c99ec7f861a42a into 54cbdc6fe7f449038158df06fdee81f4(size=30.4 K), total size for store is 60.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:58,586 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:58,586 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/A, priority=13, startTime=1731631737685; duration=0sec 2024-11-15T00:48:58,586 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:58,586 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:A 2024-11-15T00:48:58,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:58,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631798682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:58,688 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:58,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:58,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631798800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:58,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:58,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:58,812 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:58,976 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:58,981 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150d9e2acfbbd441b0a17cc2d97554d5fa_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d9e2acfbbd441b0a17cc2d97554d5fa_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:58,984 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ff7db022d1da4a988dfc63b0c4c21665, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:58,985 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ff7db022d1da4a988dfc63b0c4c21665 is 175, key is test_row_0/A:col10/1731631737900/Put/seqid=0 2024-11-15T00:48:59,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741958_1134 (size=39649) 2024-11-15T00:48:59,004 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=31.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ff7db022d1da4a988dfc63b0c4c21665 2024-11-15T00:48:59,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:59,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631799017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:59,025 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/3c123e4cc7ad4debb4c3f286331f6ece is 50, key is test_row_0/B:col10/1731631737900/Put/seqid=0 2024-11-15T00:48:59,043 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741959_1135 (size=12101) 2024-11-15T00:48:59,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/3c123e4cc7ad4debb4c3f286331f6ece 2024-11-15T00:48:59,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/5e76eff5ab7f429cb715d080b40d2d21 is 50, key is test_row_0/C:col10/1731631737900/Put/seqid=0 2024-11-15T00:48:59,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741960_1136 (size=12101) 2024-11-15T00:48:59,090 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/5e76eff5ab7f429cb715d080b40d2d21 2024-11-15T00:48:59,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ff7db022d1da4a988dfc63b0c4c21665 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ff7db022d1da4a988dfc63b0c4c21665 2024-11-15T00:48:59,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ff7db022d1da4a988dfc63b0c4c21665, entries=200, sequenceid=134, filesize=38.7 K 2024-11-15T00:48:59,125 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/3c123e4cc7ad4debb4c3f286331f6ece as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/3c123e4cc7ad4debb4c3f286331f6ece 2024-11-15T00:48:59,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/3c123e4cc7ad4debb4c3f286331f6ece, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T00:48:59,133 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/5e76eff5ab7f429cb715d080b40d2d21 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/5e76eff5ab7f429cb715d080b40d2d21 2024-11-15T00:48:59,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/5e76eff5ab7f429cb715d080b40d2d21, entries=150, sequenceid=134, filesize=11.8 K 2024-11-15T00:48:59,141 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 88182fdac0358e5518c99ec7f861a42a in 600ms, sequenceid=134, compaction requested=true 2024-11-15T00:48:59,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:59,141 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:59,142 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:48:59,142 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:59,142 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/A is initiating minor compaction (all files) 2024-11-15T00:48:59,142 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/A in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:59,143 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/54cbdc6fe7f449038158df06fdee81f4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ccf3a87dab0f4f3d8825ffc18c3cb30b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ff7db022d1da4a988dfc63b0c4c21665] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=99.4 K 2024-11-15T00:48:59,143 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:59,143 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/54cbdc6fe7f449038158df06fdee81f4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ccf3a87dab0f4f3d8825ffc18c3cb30b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ff7db022d1da4a988dfc63b0c4c21665] 2024-11-15T00:48:59,143 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 54cbdc6fe7f449038158df06fdee81f4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631735383 2024-11-15T00:48:59,144 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ccf3a87dab0f4f3d8825ffc18c3cb30b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731631737163 2024-11-15T00:48:59,144 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ff7db022d1da4a988dfc63b0c4c21665, keycount=200, bloomtype=ROW, size=38.7 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631737884 2024-11-15T00:48:59,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:59,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:48:59,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:59,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:48:59,144 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:48:59,145 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:59,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-15T00:48:59,148 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:48:59,148 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:59,148 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/B is initiating minor compaction (all files) 2024-11-15T00:48:59,148 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/B in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:59,149 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8431745a633541bd8a19dfa9654135a2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9d892d81f1bc4f9aac55e9fdc1814651, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/3c123e4cc7ad4debb4c3f286331f6ece] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=35.5 K 2024-11-15T00:48:59,149 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:48:59,150 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8431745a633541bd8a19dfa9654135a2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631735383 2024-11-15T00:48:59,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-15T00:48:59,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:48:59,152 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:48:59,153 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:48:59,153 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d892d81f1bc4f9aac55e9fdc1814651, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731631737163 2024-11-15T00:48:59,153 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:48:59,154 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3c123e4cc7ad4debb4c3f286331f6ece, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631737890 2024-11-15T00:48:59,170 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:59,172 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#B#compaction#112 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:59,172 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/0b2c2e83445f43c880715f801aa9c556 is 50, key is test_row_0/B:col10/1731631737900/Put/seqid=0 2024-11-15T00:48:59,194 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411150bdf170d4b3b4275824a204705fae7ca_88182fdac0358e5518c99ec7f861a42a store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:59,197 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411150bdf170d4b3b4275824a204705fae7ca_88182fdac0358e5518c99ec7f861a42a, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:59,197 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411150bdf170d4b3b4275824a204705fae7ca_88182fdac0358e5518c99ec7f861a42a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:59,219 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741961_1137 (size=12409) 2024-11-15T00:48:59,224 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741962_1138 (size=4469) 2024-11-15T00:48:59,226 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#A#compaction#113 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:59,227 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ceca7c51472f4796b5057a7cc23779cd is 175, key is test_row_0/A:col10/1731631737900/Put/seqid=0 2024-11-15T00:48:59,233 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/0b2c2e83445f43c880715f801aa9c556 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/0b2c2e83445f43c880715f801aa9c556 2024-11-15T00:48:59,239 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741963_1139 (size=31363) 2024-11-15T00:48:59,242 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/B of 88182fdac0358e5518c99ec7f861a42a into 0b2c2e83445f43c880715f801aa9c556(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:59,242 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:59,242 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/B, priority=13, startTime=1731631739144; duration=0sec 2024-11-15T00:48:59,242 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:48:59,242 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:B 2024-11-15T00:48:59,242 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:48:59,245 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36309 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:48:59,245 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/C is initiating minor compaction (all files) 2024-11-15T00:48:59,245 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/C in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:59,245 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/cdfd346e9ad349d4a82293680a1305b0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/3d63a125949f4ddda74a3b7f2d4b62e3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/5e76eff5ab7f429cb715d080b40d2d21] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=35.5 K 2024-11-15T00:48:59,245 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cdfd346e9ad349d4a82293680a1305b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1731631735383 2024-11-15T00:48:59,246 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3d63a125949f4ddda74a3b7f2d4b62e3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=115, earliestPutTs=1731631737163 2024-11-15T00:48:59,246 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5e76eff5ab7f429cb715d080b40d2d21, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631737890 2024-11-15T00:48:59,256 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#C#compaction#114 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:48:59,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:48:59,257 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/cd401035fa3e437a846874db1db34a85 is 50, key is test_row_0/C:col10/1731631737900/Put/seqid=0 2024-11-15T00:48:59,285 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741964_1140 (size=12409) 2024-11-15T00:48:59,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-15T00:48:59,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:48:59,307 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T00:48:59,311 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/cd401035fa3e437a846874db1db34a85 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/cd401035fa3e437a846874db1db34a85 2024-11-15T00:48:59,329 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/C of 88182fdac0358e5518c99ec7f861a42a into cd401035fa3e437a846874db1db34a85(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:59,329 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:59,329 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/C, priority=13, startTime=1731631739144; duration=0sec 2024-11-15T00:48:59,329 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:59,329 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:C 2024-11-15T00:48:59,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:59,332 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:48:59,349 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f56f89ca9b874f2f943dff9b41a81dd1_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631738609/Put/seqid=0 2024-11-15T00:48:59,366 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741965_1141 (size=12304) 2024-11-15T00:48:59,401 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:59,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631799400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,403 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:48:59,509 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:59,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631799507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,510 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:59,561 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:59,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631799556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:59,562 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 88182fdac0358e5518c99ec7f861a42a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,563 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:59,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:59,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631799567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:59,570 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 88182fdac0358e5518c99ec7f861a42a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4183 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:59,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:59,579 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:59,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631799577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:59,581 WARN [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 88182fdac0358e5518c99ec7f861a42a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,582 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:59,600 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:59,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631799597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:59,602 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 88182fdac0358e5518c99ec7f861a42a of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4214 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:48:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:59,648 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ceca7c51472f4796b5057a7cc23779cd as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ceca7c51472f4796b5057a7cc23779cd 2024-11-15T00:48:59,656 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/A of 88182fdac0358e5518c99ec7f861a42a into ceca7c51472f4796b5057a7cc23779cd(size=30.6 K), total size for store is 30.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:48:59,656 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:48:59,656 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/A, priority=13, startTime=1731631739141; duration=0sec 2024-11-15T00:48:59,656 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:48:59,656 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:A 2024-11-15T00:48:59,725 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:48:59,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631799720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:48:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:48:59,726 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:48:59,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:48:59,776 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f56f89ca9b874f2f943dff9b41a81dd1_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f56f89ca9b874f2f943dff9b41a81dd1_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:48:59,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:48:59,780 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/2a10f258f8954079a58c2215951bbdc6, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:48:59,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/2a10f258f8954079a58c2215951bbdc6 is 175, key is test_row_0/A:col10/1731631738609/Put/seqid=0 2024-11-15T00:48:59,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741966_1142 (size=31105) 2024-11-15T00:48:59,830 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=155, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/2a10f258f8954079a58c2215951bbdc6 2024-11-15T00:48:59,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/90875d72c16b4e019599ae5b53dc3f2a is 50, key is test_row_0/B:col10/1731631738609/Put/seqid=0 2024-11-15T00:48:59,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741967_1143 (size=12151) 2024-11-15T00:48:59,913 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/90875d72c16b4e019599ae5b53dc3f2a 2024-11-15T00:48:59,945 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/df8ae0c721e14401839486206c8c2c4e is 50, key is test_row_0/C:col10/1731631738609/Put/seqid=0 2024-11-15T00:48:59,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741968_1144 (size=12151) 2024-11-15T00:49:00,040 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:00,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631800037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:00,040 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:00,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:49:00,376 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/df8ae0c721e14401839486206c8c2c4e 2024-11-15T00:49:00,382 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/2a10f258f8954079a58c2215951bbdc6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2a10f258f8954079a58c2215951bbdc6 2024-11-15T00:49:00,388 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2a10f258f8954079a58c2215951bbdc6, entries=150, sequenceid=155, filesize=30.4 K 2024-11-15T00:49:00,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/90875d72c16b4e019599ae5b53dc3f2a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/90875d72c16b4e019599ae5b53dc3f2a 2024-11-15T00:49:00,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,403 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/90875d72c16b4e019599ae5b53dc3f2a, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T00:49:00,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/df8ae0c721e14401839486206c8c2c4e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/df8ae0c721e14401839486206c8c2c4e 2024-11-15T00:49:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,417 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/df8ae0c721e14401839486206c8c2c4e, entries=150, sequenceid=155, filesize=11.9 K 2024-11-15T00:49:00,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,425 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 88182fdac0358e5518c99ec7f861a42a in 1117ms, sequenceid=155, compaction requested=false 2024-11-15T00:49:00,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:00,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:00,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,429 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-15T00:49:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-15T00:49:00,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-15T00:49:00,434 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2770 sec 2024-11-15T00:49:00,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,438 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 1.2860 sec 2024-11-15T00:49:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:00,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,561 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:00,561 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T00:49:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,577 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115068aceed0f714a179a94afc8c617c83d_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631739396/Put/seqid=0 2024-11-15T00:49:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:00,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741969_1145 (size=12304) 2024-11-15T00:49:00,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:00,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631800685, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:00,690 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:00,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:00,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631800796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:00,798 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:01,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:01,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631801007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:01,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:01,048 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:01,054 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115068aceed0f714a179a94afc8c617c83d_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115068aceed0f714a179a94afc8c617c83d_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:01,055 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/0d40246c45884a0785efc40231b56363, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:01,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/0d40246c45884a0785efc40231b56363 is 175, key is test_row_0/A:col10/1731631739396/Put/seqid=0 2024-11-15T00:49:01,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741970_1146 (size=31101) 2024-11-15T00:49:01,062 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=175, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/0d40246c45884a0785efc40231b56363 2024-11-15T00:49:01,078 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/76c36715972e4d8185356418595fe1f4 is 50, key is test_row_0/B:col10/1731631739396/Put/seqid=0 2024-11-15T00:49:01,087 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741971_1147 (size=9757) 2024-11-15T00:49:01,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-15T00:49:01,297 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:01,297 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:01,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-15T00:49:01,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:49:01,299 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:01,299 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:01,299 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:01,319 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:01,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631801317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:01,320 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:01,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:49:01,451 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:49:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,451 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:01,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,452 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/76c36715972e4d8185356418595fe1f4 2024-11-15T00:49:01,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/7f3f4c14990142fc9807fbf09cf6fd78 is 50, key is test_row_0/C:col10/1731631739396/Put/seqid=0 2024-11-15T00:49:01,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741972_1148 (size=9757) 2024-11-15T00:49:01,603 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:49:01,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:01,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,604 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,604 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:49:01,756 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:49:01,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:01,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,757 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,757 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,827 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:01,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631801827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:01,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:01,903 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=175 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/7f3f4c14990142fc9807fbf09cf6fd78 2024-11-15T00:49:01,909 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/0d40246c45884a0785efc40231b56363 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/0d40246c45884a0785efc40231b56363 2024-11-15T00:49:01,910 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:49:01,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:01,910 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,911 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] handler.RSProcedureHandler(58): pid=49 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,911 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=49 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=49 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:01,916 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/0d40246c45884a0785efc40231b56363, entries=150, sequenceid=175, filesize=30.4 K 2024-11-15T00:49:01,918 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/76c36715972e4d8185356418595fe1f4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/76c36715972e4d8185356418595fe1f4 2024-11-15T00:49:01,925 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/76c36715972e4d8185356418595fe1f4, entries=100, sequenceid=175, filesize=9.5 K 2024-11-15T00:49:01,926 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/7f3f4c14990142fc9807fbf09cf6fd78 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7f3f4c14990142fc9807fbf09cf6fd78 2024-11-15T00:49:01,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:49:01,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7f3f4c14990142fc9807fbf09cf6fd78, entries=100, sequenceid=175, filesize=9.5 K 2024-11-15T00:49:01,933 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 88182fdac0358e5518c99ec7f861a42a in 1372ms, sequenceid=175, compaction requested=true 2024-11-15T00:49:01,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:01,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:01,933 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:01,933 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:01,933 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:01,935 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:01,935 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/B is initiating minor compaction (all files) 2024-11-15T00:49:01,935 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/B in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,935 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/0b2c2e83445f43c880715f801aa9c556, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/90875d72c16b4e019599ae5b53dc3f2a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/76c36715972e4d8185356418595fe1f4] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=33.5 K 2024-11-15T00:49:01,936 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93569 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:01,936 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/A is initiating minor compaction (all files) 2024-11-15T00:49:01,936 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/A in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,936 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ceca7c51472f4796b5057a7cc23779cd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2a10f258f8954079a58c2215951bbdc6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/0d40246c45884a0785efc40231b56363] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=91.4 K 2024-11-15T00:49:01,936 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:01,936 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ceca7c51472f4796b5057a7cc23779cd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2a10f258f8954079a58c2215951bbdc6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/0d40246c45884a0785efc40231b56363] 2024-11-15T00:49:01,936 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0b2c2e83445f43c880715f801aa9c556, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631737890 2024-11-15T00:49:01,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:01,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:01,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:01,937 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:01,938 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 90875d72c16b4e019599ae5b53dc3f2a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631738609 2024-11-15T00:49:01,938 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ceca7c51472f4796b5057a7cc23779cd, keycount=150, bloomtype=ROW, size=30.6 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631737890 2024-11-15T00:49:01,939 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 76c36715972e4d8185356418595fe1f4, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731631739396 2024-11-15T00:49:01,940 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a10f258f8954079a58c2215951bbdc6, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631738609 2024-11-15T00:49:01,941 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d40246c45884a0785efc40231b56363, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731631739396 2024-11-15T00:49:01,953 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:01,954 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#B#compaction#121 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:01,955 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/5f4221248f3843d29f2c88e28e5d4678 is 50, key is test_row_0/B:col10/1731631739396/Put/seqid=0 2024-11-15T00:49:01,960 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115a44cddee37784081a9072f326a0ff7cc_88182fdac0358e5518c99ec7f861a42a store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:01,962 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115a44cddee37784081a9072f326a0ff7cc_88182fdac0358e5518c99ec7f861a42a, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:01,963 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115a44cddee37784081a9072f326a0ff7cc_88182fdac0358e5518c99ec7f861a42a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:01,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741974_1150 (size=4469) 2024-11-15T00:49:01,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741973_1149 (size=12561) 2024-11-15T00:49:02,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-15T00:49:02,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:02,065 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T00:49:02,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f05113efe7d4075a27b2a9d5ebdff44_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631740653/Put/seqid=0 2024-11-15T00:49:02,078 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741975_1151 (size=12304) 2024-11-15T00:49:02,389 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#A#compaction#122 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:02,392 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/a5203fbe096f4f8fa886172131be6383 is 175, key is test_row_0/A:col10/1731631739396/Put/seqid=0 2024-11-15T00:49:02,403 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/5f4221248f3843d29f2c88e28e5d4678 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/5f4221248f3843d29f2c88e28e5d4678 2024-11-15T00:49:02,417 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/B of 88182fdac0358e5518c99ec7f861a42a into 5f4221248f3843d29f2c88e28e5d4678(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:02,417 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:02,417 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/B, priority=13, startTime=1731631741933; duration=0sec 2024-11-15T00:49:02,417 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:02,417 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:B 2024-11-15T00:49:02,417 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:02,419 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34317 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:02,419 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/C is initiating minor compaction (all files) 2024-11-15T00:49:02,419 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/C in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:02,419 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/cd401035fa3e437a846874db1db34a85, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/df8ae0c721e14401839486206c8c2c4e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7f3f4c14990142fc9807fbf09cf6fd78] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=33.5 K 2024-11-15T00:49:02,420 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting cd401035fa3e437a846874db1db34a85, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1731631737890 2024-11-15T00:49:02,421 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting df8ae0c721e14401839486206c8c2c4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1731631738609 2024-11-15T00:49:02,422 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f3f4c14990142fc9807fbf09cf6fd78, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731631739396 2024-11-15T00:49:02,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741976_1152 (size=31622) 2024-11-15T00:49:02,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:49:02,462 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#C#compaction#124 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:02,463 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/8dc14e7d7d864cbfa050faabb16cf44c is 50, key is test_row_0/C:col10/1731631739396/Put/seqid=0 2024-11-15T00:49:02,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:02,489 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741977_1153 (size=12561) 2024-11-15T00:49:02,505 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153f05113efe7d4075a27b2a9d5ebdff44_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f05113efe7d4075a27b2a9d5ebdff44_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:02,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ea3cde91d30b4ea7869a3952ebc25eae, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:02,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ea3cde91d30b4ea7869a3952ebc25eae is 175, key is test_row_0/A:col10/1731631740653/Put/seqid=0 2024-11-15T00:49:02,515 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/8dc14e7d7d864cbfa050faabb16cf44c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/8dc14e7d7d864cbfa050faabb16cf44c 2024-11-15T00:49:02,524 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/C of 88182fdac0358e5518c99ec7f861a42a into 8dc14e7d7d864cbfa050faabb16cf44c(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:02,525 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:02,525 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/C, priority=13, startTime=1731631741937; duration=0sec 2024-11-15T00:49:02,525 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:02,525 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:C 2024-11-15T00:49:02,535 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741978_1154 (size=31105) 2024-11-15T00:49:02,833 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/a5203fbe096f4f8fa886172131be6383 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/a5203fbe096f4f8fa886172131be6383 2024-11-15T00:49:02,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:02,843 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:02,851 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/A of 88182fdac0358e5518c99ec7f861a42a into a5203fbe096f4f8fa886172131be6383(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:02,851 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:02,851 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/A, priority=13, startTime=1731631741933; duration=0sec 2024-11-15T00:49:02,851 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:02,851 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:A 2024-11-15T00:49:02,934 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=193, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ea3cde91d30b4ea7869a3952ebc25eae 2024-11-15T00:49:02,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/9f358eb6721f4665818c15454810330c is 50, key is test_row_0/B:col10/1731631740653/Put/seqid=0 2024-11-15T00:49:02,975 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:02,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631802971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:02,976 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:02,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741979_1155 (size=12151) 2024-11-15T00:49:02,981 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/9f358eb6721f4665818c15454810330c 2024-11-15T00:49:03,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/008bd2d0f9d04eb2a737e220a5985010 is 50, key is test_row_0/C:col10/1731631740653/Put/seqid=0 2024-11-15T00:49:03,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741980_1156 (size=12151) 2024-11-15T00:49:03,090 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631803088, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,092 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,308 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631803301, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,311 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,432 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=193 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/008bd2d0f9d04eb2a737e220a5985010 2024-11-15T00:49:03,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/ea3cde91d30b4ea7869a3952ebc25eae as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ea3cde91d30b4ea7869a3952ebc25eae 2024-11-15T00:49:03,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:49:03,452 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ea3cde91d30b4ea7869a3952ebc25eae, entries=150, sequenceid=193, filesize=30.4 K 2024-11-15T00:49:03,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/9f358eb6721f4665818c15454810330c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9f358eb6721f4665818c15454810330c 2024-11-15T00:49:03,462 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9f358eb6721f4665818c15454810330c, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T00:49:03,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/008bd2d0f9d04eb2a737e220a5985010 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/008bd2d0f9d04eb2a737e220a5985010 2024-11-15T00:49:03,474 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/008bd2d0f9d04eb2a737e220a5985010, entries=150, sequenceid=193, filesize=11.9 K 2024-11-15T00:49:03,485 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 88182fdac0358e5518c99ec7f861a42a in 1421ms, sequenceid=193, compaction requested=false 2024-11-15T00:49:03,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:03,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:03,485 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-15T00:49:03,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-15T00:49:03,489 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-15T00:49:03,490 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1880 sec 2024-11-15T00:49:03,496 INFO [master/fb2f6cb7846a:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-15T00:49:03,496 INFO [master/fb2f6cb7846a:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-15T00:49:03,502 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 2.1940 sec 2024-11-15T00:49:03,589 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:03,589 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T00:49:03,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:03,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ddb2e018e079432d8b4af951451f3b17_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631743581/Put/seqid=0 2024-11-15T00:49:03,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741981_1157 (size=12304) 2024-11-15T00:49:03,660 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:03,667 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ddb2e018e079432d8b4af951451f3b17_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ddb2e018e079432d8b4af951451f3b17_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:03,669 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/2f3cfc9f15b447e6bcae007a4fffb40b, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:03,670 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/2f3cfc9f15b447e6bcae007a4fffb40b is 175, key is test_row_0/A:col10/1731631743581/Put/seqid=0 2024-11-15T00:49:03,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631803670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,689 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631803675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,689 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631803672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,690 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,691 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631803688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,692 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,692 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631803689, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,700 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,714 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741982_1158 (size=31105) 2024-11-15T00:49:03,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631803798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,805 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,805 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631803798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,806 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,807 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631803798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631803798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:03,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:03,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631803807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:03,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,817 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:03,818 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631804020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,024 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,024 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631804021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,025 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631804021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631804021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,026 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,028 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631804029, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,115 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=215, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/2f3cfc9f15b447e6bcae007a4fffb40b 2024-11-15T00:49:04,147 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/681c89297fc94910ac7dccb878121578 is 50, key is test_row_0/B:col10/1731631743581/Put/seqid=0 2024-11-15T00:49:04,156 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741983_1159 (size=12151) 2024-11-15T00:49:04,327 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631804327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,328 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,342 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631804337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631804337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,343 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,343 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631804337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631804337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,343 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,344 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,561 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/681c89297fc94910ac7dccb878121578 2024-11-15T00:49:04,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/7778e5e326f64ec69fd3202d6be62e45 is 50, key is test_row_0/C:col10/1731631743581/Put/seqid=0 2024-11-15T00:49:04,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741984_1160 (size=12151) 2024-11-15T00:49:04,640 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=215 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/7778e5e326f64ec69fd3202d6be62e45 2024-11-15T00:49:04,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/2f3cfc9f15b447e6bcae007a4fffb40b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2f3cfc9f15b447e6bcae007a4fffb40b 2024-11-15T00:49:04,668 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2f3cfc9f15b447e6bcae007a4fffb40b, entries=150, sequenceid=215, filesize=30.4 K 2024-11-15T00:49:04,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/681c89297fc94910ac7dccb878121578 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/681c89297fc94910ac7dccb878121578 2024-11-15T00:49:04,691 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/681c89297fc94910ac7dccb878121578, entries=150, sequenceid=215, filesize=11.9 K 2024-11-15T00:49:04,696 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/7778e5e326f64ec69fd3202d6be62e45 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7778e5e326f64ec69fd3202d6be62e45 2024-11-15T00:49:04,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7778e5e326f64ec69fd3202d6be62e45, entries=150, sequenceid=215, filesize=11.9 K 2024-11-15T00:49:04,736 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 88182fdac0358e5518c99ec7f861a42a in 1146ms, sequenceid=215, compaction requested=true 2024-11-15T00:49:04,736 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:04,736 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:04,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:04,736 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:04,737 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:04,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:04,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:04,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:04,737 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:04,738 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93832 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:04,738 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/A is initiating minor compaction (all files) 2024-11-15T00:49:04,738 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/A in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:04,739 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/a5203fbe096f4f8fa886172131be6383, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ea3cde91d30b4ea7869a3952ebc25eae, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2f3cfc9f15b447e6bcae007a4fffb40b] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=91.6 K 2024-11-15T00:49:04,739 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:04,739 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/a5203fbe096f4f8fa886172131be6383, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ea3cde91d30b4ea7869a3952ebc25eae, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2f3cfc9f15b447e6bcae007a4fffb40b] 2024-11-15T00:49:04,741 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5203fbe096f4f8fa886172131be6383, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731631738678 2024-11-15T00:49:04,742 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:04,742 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/B is initiating minor compaction (all files) 2024-11-15T00:49:04,742 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/B in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:04,742 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/5f4221248f3843d29f2c88e28e5d4678, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9f358eb6721f4665818c15454810330c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/681c89297fc94910ac7dccb878121578] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=36.0 K 2024-11-15T00:49:04,742 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ea3cde91d30b4ea7869a3952ebc25eae, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731631740653 2024-11-15T00:49:04,742 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f4221248f3843d29f2c88e28e5d4678, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731631738678 2024-11-15T00:49:04,744 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f358eb6721f4665818c15454810330c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731631740653 2024-11-15T00:49:04,744 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2f3cfc9f15b447e6bcae007a4fffb40b, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731631742935 2024-11-15T00:49:04,745 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 681c89297fc94910ac7dccb878121578, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731631742935 2024-11-15T00:49:04,773 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#B#compaction#130 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:04,774 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/995a6a196dab4a0296bd54511bcc5d78 is 50, key is test_row_0/B:col10/1731631743581/Put/seqid=0 2024-11-15T00:49:04,785 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:04,797 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e5f1cd8e995e4d8d9c38337d8574d913_88182fdac0358e5518c99ec7f861a42a store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:04,799 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e5f1cd8e995e4d8d9c38337d8574d913_88182fdac0358e5518c99ec7f861a42a, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:04,799 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e5f1cd8e995e4d8d9c38337d8574d913_88182fdac0358e5518c99ec7f861a42a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:04,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741985_1161 (size=12663) 2024-11-15T00:49:04,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741986_1162 (size=4469) 2024-11-15T00:49:04,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:04,846 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:04,847 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T00:49:04,858 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/995a6a196dab4a0296bd54511bcc5d78 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/995a6a196dab4a0296bd54511bcc5d78 2024-11-15T00:49:04,869 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/B of 88182fdac0358e5518c99ec7f861a42a into 995a6a196dab4a0296bd54511bcc5d78(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:04,872 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:04,872 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/B, priority=13, startTime=1731631744736; duration=0sec 2024-11-15T00:49:04,874 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:04,874 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:B 2024-11-15T00:49:04,874 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:04,876 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:04,877 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/C is initiating minor compaction (all files) 2024-11-15T00:49:04,877 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/C in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:04,880 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/8dc14e7d7d864cbfa050faabb16cf44c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/008bd2d0f9d04eb2a737e220a5985010, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7778e5e326f64ec69fd3202d6be62e45] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=36.0 K 2024-11-15T00:49:04,881 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dc14e7d7d864cbfa050faabb16cf44c, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=175, earliestPutTs=1731631738678 2024-11-15T00:49:04,885 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 008bd2d0f9d04eb2a737e220a5985010, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=193, earliestPutTs=1731631740653 2024-11-15T00:49:04,889 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7778e5e326f64ec69fd3202d6be62e45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731631742935 2024-11-15T00:49:04,893 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bf4ab97b5bc24ad6971d358b8a16c0f8_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631743685/Put/seqid=0 2024-11-15T00:49:04,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631804896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631804898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,906 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,906 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631804900, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,907 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631804901, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,908 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:04,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631804904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:04,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:04,943 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#C#compaction#133 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:04,943 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/36d114f6f7f5432187c04bff583bb5fa is 50, key is test_row_0/C:col10/1731631743581/Put/seqid=0 2024-11-15T00:49:04,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741987_1163 (size=14794) 2024-11-15T00:49:04,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741988_1164 (size=12663) 2024-11-15T00:49:05,000 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/36d114f6f7f5432187c04bff583bb5fa as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/36d114f6f7f5432187c04bff583bb5fa 2024-11-15T00:49:05,008 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/C of 88182fdac0358e5518c99ec7f861a42a into 36d114f6f7f5432187c04bff583bb5fa(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:05,009 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:05,009 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/C, priority=13, startTime=1731631744737; duration=0sec 2024-11-15T00:49:05,009 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:05,009 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:C 2024-11-15T00:49:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631805018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631805019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631805020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,024 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631805023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631805023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,032 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,229 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631805226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631805226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,238 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#A#compaction#131 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:05,239 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/798ef5f7609d497c9a2135f0fbf9770a is 175, key is test_row_0/A:col10/1731631743581/Put/seqid=0 2024-11-15T00:49:05,241 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631805237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,242 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,242 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631805237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631805237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,242 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741989_1165 (size=31617) 2024-11-15T00:49:05,275 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/798ef5f7609d497c9a2135f0fbf9770a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/798ef5f7609d497c9a2135f0fbf9770a 2024-11-15T00:49:05,293 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/A of 88182fdac0358e5518c99ec7f861a42a into 798ef5f7609d497c9a2135f0fbf9770a(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:05,293 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:05,293 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/A, priority=13, startTime=1731631744736; duration=0sec 2024-11-15T00:49:05,293 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:05,293 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:A 2024-11-15T00:49:05,351 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:05,356 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bf4ab97b5bc24ad6971d358b8a16c0f8_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf4ab97b5bc24ad6971d358b8a16c0f8_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:05,357 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/6da95f8967f44298becb51840a5f27d9, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:05,358 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/6da95f8967f44298becb51840a5f27d9 is 175, key is test_row_0/A:col10/1731631743685/Put/seqid=0 2024-11-15T00:49:05,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741990_1166 (size=39749) 2024-11-15T00:49:05,366 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/6da95f8967f44298becb51840a5f27d9 2024-11-15T00:49:05,375 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/02ba5f1f92f343278161b67b4dc8bca6 is 50, key is test_row_0/B:col10/1731631743685/Put/seqid=0 2024-11-15T00:49:05,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741991_1167 (size=12151) 2024-11-15T00:49:05,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-15T00:49:05,457 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:05,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:05,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-15T00:49:05,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:49:05,460 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:05,469 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:05,469 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:05,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631805540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,542 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,542 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631805540, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,543 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631805546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,550 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631805546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:05,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631805548, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:05,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,551 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:05,552 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:49:05,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T00:49:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:05,629 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:05,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:05,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:05,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:49:05,793 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T00:49:05,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:05,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:05,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:05,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:05,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:05,794 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/02ba5f1f92f343278161b67b4dc8bca6 2024-11-15T00:49:05,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:05,837 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/fad6578f0b1349c387ba3d0e6a875dc3 is 50, key is test_row_0/C:col10/1731631743685/Put/seqid=0 2024-11-15T00:49:05,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741992_1168 (size=12151) 2024-11-15T00:49:05,893 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/fad6578f0b1349c387ba3d0e6a875dc3 2024-11-15T00:49:05,899 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/6da95f8967f44298becb51840a5f27d9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/6da95f8967f44298becb51840a5f27d9 2024-11-15T00:49:05,906 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/6da95f8967f44298becb51840a5f27d9, entries=200, sequenceid=233, filesize=38.8 K 2024-11-15T00:49:05,907 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/02ba5f1f92f343278161b67b4dc8bca6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/02ba5f1f92f343278161b67b4dc8bca6 2024-11-15T00:49:05,915 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/02ba5f1f92f343278161b67b4dc8bca6, entries=150, sequenceid=233, filesize=11.9 K 2024-11-15T00:49:05,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/fad6578f0b1349c387ba3d0e6a875dc3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/fad6578f0b1349c387ba3d0e6a875dc3 2024-11-15T00:49:05,924 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/fad6578f0b1349c387ba3d0e6a875dc3, entries=150, sequenceid=233, filesize=11.9 K 2024-11-15T00:49:05,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 88182fdac0358e5518c99ec7f861a42a in 1079ms, sequenceid=233, compaction requested=false 2024-11-15T00:49:05,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:05,948 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-15T00:49:05,948 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:05,948 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T00:49:05,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111544830d7445f741abb18d9c02ff8e9b12_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631744897/Put/seqid=0 2024-11-15T00:49:05,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741993_1169 (size=12304) 2024-11-15T00:49:06,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:06,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:06,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:49:06,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631806084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,092 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,096 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631806085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,097 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,097 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631806089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631806090, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,098 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,099 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631806091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,099 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631806202, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,207 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,208 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,211 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631806207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631806207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631806207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,212 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,213 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631806208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,221 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:06,400 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111544830d7445f741abb18d9c02ff8e9b12_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111544830d7445f741abb18d9c02ff8e9b12_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:06,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/f736e0f99b154e75ac635837a4c992c0, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:06,402 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/f736e0f99b154e75ac635837a4c992c0 is 175, key is test_row_0/A:col10/1731631744897/Put/seqid=0 2024-11-15T00:49:06,407 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741994_1170 (size=31105) 2024-11-15T00:49:06,409 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=254, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/f736e0f99b154e75ac635837a4c992c0 2024-11-15T00:49:06,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/7d584fef372e4f84b2b7e858b329abe4 is 50, key is test_row_0/B:col10/1731631744897/Put/seqid=0 2024-11-15T00:49:06,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631806417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631806418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,422 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631806420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631806420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,429 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631806429, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,430 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741995_1171 (size=12151) 2024-11-15T00:49:06,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:49:06,730 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631806726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631806726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,731 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631806737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631806737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,738 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631806737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:06,739 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:06,859 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/7d584fef372e4f84b2b7e858b329abe4 2024-11-15T00:49:06,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/1b3b3c2f6089478fad7d8988c5f9769e is 50, key is test_row_0/C:col10/1731631744897/Put/seqid=0 2024-11-15T00:49:06,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741996_1172 (size=12151) 2024-11-15T00:49:06,884 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=254 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/1b3b3c2f6089478fad7d8988c5f9769e 2024-11-15T00:49:06,889 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/f736e0f99b154e75ac635837a4c992c0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/f736e0f99b154e75ac635837a4c992c0 2024-11-15T00:49:06,904 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/f736e0f99b154e75ac635837a4c992c0, entries=150, sequenceid=254, filesize=30.4 K 2024-11-15T00:49:06,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/7d584fef372e4f84b2b7e858b329abe4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/7d584fef372e4f84b2b7e858b329abe4 2024-11-15T00:49:06,934 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/7d584fef372e4f84b2b7e858b329abe4, entries=150, sequenceid=254, filesize=11.9 K 2024-11-15T00:49:06,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/1b3b3c2f6089478fad7d8988c5f9769e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/1b3b3c2f6089478fad7d8988c5f9769e 2024-11-15T00:49:06,946 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/1b3b3c2f6089478fad7d8988c5f9769e, entries=150, sequenceid=254, filesize=11.9 K 2024-11-15T00:49:06,947 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 88182fdac0358e5518c99ec7f861a42a in 999ms, sequenceid=254, compaction requested=true 2024-11-15T00:49:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-15T00:49:06,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-15T00:49:06,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-15T00:49:06,955 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4830 sec 2024-11-15T00:49:06,960 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 1.5000 sec 2024-11-15T00:49:07,241 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:07,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T00:49:07,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:07,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d99b610e18f646e0a8e5393ebf4e4166_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631746089/Put/seqid=0 2024-11-15T00:49:07,281 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631807274, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,282 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631807280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,288 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631807282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741997_1173 (size=14994) 2024-11-15T00:49:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631807284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,290 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631807284, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,293 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631807387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,403 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631807397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631807397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631807397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,405 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,405 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631807397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,408 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,409 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631807597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,599 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-15T00:49:07,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,618 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631807617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631807617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,618 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:07,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631807618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,618 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,619 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631807617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-15T00:49:07,619 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T00:49:07,621 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:07,621 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:07,622 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:07,691 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:07,698 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115d99b610e18f646e0a8e5393ebf4e4166_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d99b610e18f646e0a8e5393ebf4e4166_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:07,699 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/185a794ee9214dcb951227fb65755f79, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:07,700 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/185a794ee9214dcb951227fb65755f79 is 175, key is test_row_0/A:col10/1731631746089/Put/seqid=0 2024-11-15T00:49:07,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T00:49:07,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741998_1174 (size=39949) 2024-11-15T00:49:07,774 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T00:49:07,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:07,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:07,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:07,775 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:07,775 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:07,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:07,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631807906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,907 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631807927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,934 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631807927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,934 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,935 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631807927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T00:49:07,935 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:07,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631807927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:07,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:07,935 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:07,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:07,936 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:07,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T00:49:07,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,093 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T00:49:08,093 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:08,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,140 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=273, memsize=35.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/185a794ee9214dcb951227fb65755f79 2024-11-15T00:49:08,150 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/b3f9c258d9934a43a30c7dc3303cc96a is 50, key is test_row_0/B:col10/1731631746089/Put/seqid=0 2024-11-15T00:49:08,154 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741999_1175 (size=12301) 2024-11-15T00:49:08,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T00:49:08,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T00:49:08,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:08,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,249 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,249 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T00:49:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:08,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:08,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631808416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:08,421 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:08,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:08,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631808447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:08,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:08,451 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631808447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:08,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631808447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:08,452 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:08,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:08,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631808448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:08,453 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:08,555 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/b3f9c258d9934a43a30c7dc3303cc96a 2024-11-15T00:49:08,565 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/39952c0024044b66a94fe262629bc983 is 50, key is test_row_0/C:col10/1731631746089/Put/seqid=0 2024-11-15T00:49:08,569 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T00:49:08,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:08,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,569 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742000_1176 (size=12301) 2024-11-15T00:49:08,727 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T00:49:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,727 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,727 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T00:49:08,881 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T00:49:08,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:08,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:08,882 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:08,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=273 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/39952c0024044b66a94fe262629bc983 2024-11-15T00:49:09,003 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/185a794ee9214dcb951227fb65755f79 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/185a794ee9214dcb951227fb65755f79 2024-11-15T00:49:09,010 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/185a794ee9214dcb951227fb65755f79, entries=200, sequenceid=273, filesize=39.0 K 2024-11-15T00:49:09,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/b3f9c258d9934a43a30c7dc3303cc96a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b3f9c258d9934a43a30c7dc3303cc96a 2024-11-15T00:49:09,017 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b3f9c258d9934a43a30c7dc3303cc96a, entries=150, sequenceid=273, filesize=12.0 K 2024-11-15T00:49:09,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/39952c0024044b66a94fe262629bc983 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/39952c0024044b66a94fe262629bc983 2024-11-15T00:49:09,024 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/39952c0024044b66a94fe262629bc983, entries=150, sequenceid=273, filesize=12.0 K 2024-11-15T00:49:09,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=100.63 KB/103050 for 88182fdac0358e5518c99ec7f861a42a in 1784ms, sequenceid=273, compaction requested=true 2024-11-15T00:49:09,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:09,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:09,025 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:09,025 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:09,026 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:09,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:09,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:09,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:09,026 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:09,027 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142420 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:09,027 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:09,027 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/A is initiating minor compaction (all files) 2024-11-15T00:49:09,027 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/B is initiating minor compaction (all files) 2024-11-15T00:49:09,027 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/A in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:09,027 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/B in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:09,027 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/798ef5f7609d497c9a2135f0fbf9770a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/6da95f8967f44298becb51840a5f27d9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/f736e0f99b154e75ac635837a4c992c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/185a794ee9214dcb951227fb65755f79] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=139.1 K 2024-11-15T00:49:09,027 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/995a6a196dab4a0296bd54511bcc5d78, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/02ba5f1f92f343278161b67b4dc8bca6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/7d584fef372e4f84b2b7e858b329abe4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b3f9c258d9934a43a30c7dc3303cc96a] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=48.1 K 2024-11-15T00:49:09,027 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:09,028 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 995a6a196dab4a0296bd54511bcc5d78, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731631742935 2024-11-15T00:49:09,028 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 02ba5f1f92f343278161b67b4dc8bca6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731631743659 2024-11-15T00:49:09,028 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d584fef372e4f84b2b7e858b329abe4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731631744883 2024-11-15T00:49:09,029 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b3f9c258d9934a43a30c7dc3303cc96a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731631746085 2024-11-15T00:49:09,027 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/798ef5f7609d497c9a2135f0fbf9770a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/6da95f8967f44298becb51840a5f27d9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/f736e0f99b154e75ac635837a4c992c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/185a794ee9214dcb951227fb65755f79] 2024-11-15T00:49:09,031 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 798ef5f7609d497c9a2135f0fbf9770a, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731631742935 2024-11-15T00:49:09,032 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6da95f8967f44298becb51840a5f27d9, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731631743659 2024-11-15T00:49:09,032 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f736e0f99b154e75ac635837a4c992c0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731631744883 2024-11-15T00:49:09,033 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 185a794ee9214dcb951227fb65755f79, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731631746085 2024-11-15T00:49:09,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-15T00:49:09,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:09,040 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-15T00:49:09,042 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#B#compaction#142 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:09,042 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/66d9a9ed2cc0491ab98e9301883027c7 is 50, key is test_row_0/B:col10/1731631746089/Put/seqid=0 2024-11-15T00:49:09,054 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:09,056 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742001_1177 (size=12949) 2024-11-15T00:49:09,058 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153a6d8acdc1234765ad35c55f9194c870_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631747279/Put/seqid=0 2024-11-15T00:49:09,062 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ee8de788b52b4ddf82262e14647c28c3_88182fdac0358e5518c99ec7f861a42a store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:09,065 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ee8de788b52b4ddf82262e14647c28c3_88182fdac0358e5518c99ec7f861a42a, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:09,065 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ee8de788b52b4ddf82262e14647c28c3_88182fdac0358e5518c99ec7f861a42a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:09,093 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742002_1178 (size=12454) 2024-11-15T00:49:09,102 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742003_1179 (size=4469) 2024-11-15T00:49:09,109 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#A#compaction#143 average throughput is 0.46 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:09,109 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/d53954d2e24948d980d5d8b78e031d8a is 175, key is test_row_0/A:col10/1731631746089/Put/seqid=0 2024-11-15T00:49:09,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742004_1180 (size=31903) 2024-11-15T00:49:09,132 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/d53954d2e24948d980d5d8b78e031d8a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/d53954d2e24948d980d5d8b78e031d8a 2024-11-15T00:49:09,146 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/A of 88182fdac0358e5518c99ec7f861a42a into d53954d2e24948d980d5d8b78e031d8a(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:09,146 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:09,146 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/A, priority=12, startTime=1731631749025; duration=0sec 2024-11-15T00:49:09,146 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:09,146 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:A 2024-11-15T00:49:09,146 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:09,148 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:09,148 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/C is initiating minor compaction (all files) 2024-11-15T00:49:09,149 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/C in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:09,149 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/36d114f6f7f5432187c04bff583bb5fa, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/fad6578f0b1349c387ba3d0e6a875dc3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/1b3b3c2f6089478fad7d8988c5f9769e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/39952c0024044b66a94fe262629bc983] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=48.1 K 2024-11-15T00:49:09,149 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36d114f6f7f5432187c04bff583bb5fa, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=215, earliestPutTs=1731631742935 2024-11-15T00:49:09,150 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting fad6578f0b1349c387ba3d0e6a875dc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731631743659 2024-11-15T00:49:09,150 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1b3b3c2f6089478fad7d8988c5f9769e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=254, earliestPutTs=1731631744883 2024-11-15T00:49:09,150 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 39952c0024044b66a94fe262629bc983, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731631746085 2024-11-15T00:49:09,162 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#C#compaction#145 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:09,163 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/962b1ec1d5c74526a629a0bdbcfcf7c0 is 50, key is test_row_0/C:col10/1731631746089/Put/seqid=0 2024-11-15T00:49:09,169 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742005_1181 (size=12949) 2024-11-15T00:49:09,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:09,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:09,480 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/66d9a9ed2cc0491ab98e9301883027c7 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/66d9a9ed2cc0491ab98e9301883027c7 2024-11-15T00:49:09,495 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631809489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,495 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631809489, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,496 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,497 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/B of 88182fdac0358e5518c99ec7f861a42a into 66d9a9ed2cc0491ab98e9301883027c7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:09,497 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:09,497 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/B, priority=12, startTime=1731631749025; duration=0sec 2024-11-15T00:49:09,497 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:09,497 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:B 2024-11-15T00:49:09,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631809492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,502 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,507 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153a6d8acdc1234765ad35c55f9194c870_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a6d8acdc1234765ad35c55f9194c870_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:09,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/5b337921b9cf436b9e241f036ac7354e, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:09,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631809495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/5b337921b9cf436b9e241f036ac7354e is 175, key is test_row_0/A:col10/1731631747279/Put/seqid=0 2024-11-15T00:49:09,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,509 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,509 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631809495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,509 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742006_1182 (size=31255) 2024-11-15T00:49:09,601 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/962b1ec1d5c74526a629a0bdbcfcf7c0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/962b1ec1d5c74526a629a0bdbcfcf7c0 2024-11-15T00:49:09,607 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631809607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631809607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,608 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631809608, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,616 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,617 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631809619, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631809620, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,638 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/C of 88182fdac0358e5518c99ec7f861a42a into 962b1ec1d5c74526a629a0bdbcfcf7c0(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:09,638 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:09,638 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/C, priority=12, startTime=1731631749026; duration=0sec 2024-11-15T00:49:09,638 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:09,638 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:C 2024-11-15T00:49:09,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T00:49:09,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631809816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,817 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631809817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,820 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,830 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631809827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,831 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,840 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631809837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,841 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631809837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:09,841 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:09,954 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=33.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/5b337921b9cf436b9e241f036ac7354e 2024-11-15T00:49:09,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/e5f420acb2c94753a468ebbb0b46549c is 50, key is test_row_0/B:col10/1731631747279/Put/seqid=0 2024-11-15T00:49:09,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742007_1183 (size=12301) 2024-11-15T00:49:10,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631810126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,127 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631810126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,127 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,130 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,137 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631810137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,138 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631810148, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631810149, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,150 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,367 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/e5f420acb2c94753a468ebbb0b46549c 2024-11-15T00:49:10,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/ba95d647d69943f9aac3e7a790bf4280 is 50, key is test_row_0/C:col10/1731631747279/Put/seqid=0 2024-11-15T00:49:10,385 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742008_1184 (size=12301) 2024-11-15T00:49:10,386 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/ba95d647d69943f9aac3e7a790bf4280 2024-11-15T00:49:10,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/5b337921b9cf436b9e241f036ac7354e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/5b337921b9cf436b9e241f036ac7354e 2024-11-15T00:49:10,412 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/5b337921b9cf436b9e241f036ac7354e, entries=150, sequenceid=291, filesize=30.5 K 2024-11-15T00:49:10,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/e5f420acb2c94753a468ebbb0b46549c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e5f420acb2c94753a468ebbb0b46549c 2024-11-15T00:49:10,450 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e5f420acb2c94753a468ebbb0b46549c, entries=150, sequenceid=291, filesize=12.0 K 2024-11-15T00:49:10,452 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/ba95d647d69943f9aac3e7a790bf4280 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/ba95d647d69943f9aac3e7a790bf4280 2024-11-15T00:49:10,459 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/ba95d647d69943f9aac3e7a790bf4280, entries=150, sequenceid=291, filesize=12.0 K 2024-11-15T00:49:10,461 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 88182fdac0358e5518c99ec7f861a42a in 1421ms, sequenceid=291, compaction requested=false 2024-11-15T00:49:10,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:10,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:10,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-15T00:49:10,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-15T00:49:10,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-15T00:49:10,465 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8420 sec 2024-11-15T00:49:10,467 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 2.8480 sec 2024-11-15T00:49:10,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:10,641 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:10,641 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-15T00:49:10,675 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631810669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631810670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631810670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631810671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631810671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111549fafbdbc0f4450e8157591283546062_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631749492/Put/seqid=0 2024-11-15T00:49:10,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742009_1185 (size=12454) 2024-11-15T00:49:10,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631810787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631810788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,789 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631810788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631810788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,790 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631810788, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,790 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,791 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,997 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631810997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631810997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631810997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,998 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,998 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:10,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631810997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:10,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,001 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631810998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,001 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,136 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:11,140 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111549fafbdbc0f4450e8157591283546062_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111549fafbdbc0f4450e8157591283546062_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:11,144 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/1644cf9b1c8c48d5956dd862a8d136b1, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:11,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/1644cf9b1c8c48d5956dd862a8d136b1 is 175, key is test_row_0/A:col10/1731631749492/Put/seqid=0 2024-11-15T00:49:11,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742010_1186 (size=31255) 2024-11-15T00:49:11,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631811307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631811307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631811307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631811307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,312 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631811308, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,312 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,313 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,313 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,551 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=314, memsize=38.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/1644cf9b1c8c48d5956dd862a8d136b1 2024-11-15T00:49:11,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/b48ff679700744688a546a2b5ec19e5a is 50, key is test_row_0/B:col10/1731631749492/Put/seqid=0 2024-11-15T00:49:11,574 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742011_1187 (size=12301) 2024-11-15T00:49:11,575 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/b48ff679700744688a546a2b5ec19e5a 2024-11-15T00:49:11,584 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/6ea6859292cc405e9904c8dc47771f7f is 50, key is test_row_0/C:col10/1731631749492/Put/seqid=0 2024-11-15T00:49:11,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742012_1188 (size=12301) 2024-11-15T00:49:11,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-15T00:49:11,787 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:11,787 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:11,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-15T00:49:11,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T00:49:11,789 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:11,789 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:11,789 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:11,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35634 deadline: 1731631811817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35596 deadline: 1731631811818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,819 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35612 deadline: 1731631811818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,820 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35586 deadline: 1731631811824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,825 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:11,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:35626 deadline: 1731631811825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,826 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:11,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T00:49:11,941 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T00:49:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:11,942 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:11,942 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:11,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:12,048 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/6ea6859292cc405e9904c8dc47771f7f 2024-11-15T00:49:12,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/1644cf9b1c8c48d5956dd862a8d136b1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/1644cf9b1c8c48d5956dd862a8d136b1 2024-11-15T00:49:12,064 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/1644cf9b1c8c48d5956dd862a8d136b1, entries=150, sequenceid=314, filesize=30.5 K 2024-11-15T00:49:12,077 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/b48ff679700744688a546a2b5ec19e5a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b48ff679700744688a546a2b5ec19e5a 2024-11-15T00:49:12,089 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b48ff679700744688a546a2b5ec19e5a, entries=150, sequenceid=314, filesize=12.0 K 2024-11-15T00:49:12,091 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/6ea6859292cc405e9904c8dc47771f7f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6ea6859292cc405e9904c8dc47771f7f 2024-11-15T00:49:12,095 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T00:49:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. as already flushing 2024-11-15T00:49:12,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:12,096 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:12,096 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:12,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6ea6859292cc405e9904c8dc47771f7f, entries=150, sequenceid=314, filesize=12.0 K 2024-11-15T00:49:12,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:12,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 88182fdac0358e5518c99ec7f861a42a in 1456ms, sequenceid=314, compaction requested=true 2024-11-15T00:49:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:12,098 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:12,098 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 88182fdac0358e5518c99ec7f861a42a:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:12,098 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:12,099 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94413 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:12,099 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:12,099 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/A is initiating minor compaction (all files) 2024-11-15T00:49:12,099 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/B is initiating minor compaction (all files) 2024-11-15T00:49:12,099 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/B in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:12,099 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/A in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:12,099 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/66d9a9ed2cc0491ab98e9301883027c7, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e5f420acb2c94753a468ebbb0b46549c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b48ff679700744688a546a2b5ec19e5a] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=36.7 K 2024-11-15T00:49:12,099 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/d53954d2e24948d980d5d8b78e031d8a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/5b337921b9cf436b9e241f036ac7354e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/1644cf9b1c8c48d5956dd862a8d136b1] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=92.2 K 2024-11-15T00:49:12,099 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:12,100 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/d53954d2e24948d980d5d8b78e031d8a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/5b337921b9cf436b9e241f036ac7354e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/1644cf9b1c8c48d5956dd862a8d136b1] 2024-11-15T00:49:12,102 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d53954d2e24948d980d5d8b78e031d8a, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731631746085 2024-11-15T00:49:12,102 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 66d9a9ed2cc0491ab98e9301883027c7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731631746085 2024-11-15T00:49:12,102 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e5f420acb2c94753a468ebbb0b46549c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731631747277 2024-11-15T00:49:12,102 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5b337921b9cf436b9e241f036ac7354e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731631747277 2024-11-15T00:49:12,103 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b48ff679700744688a546a2b5ec19e5a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731631749492 2024-11-15T00:49:12,103 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1644cf9b1c8c48d5956dd862a8d136b1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731631749492 2024-11-15T00:49:12,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T00:49:12,117 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:12,139 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#B#compaction#152 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:12,139 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/c75fb14d546e45a3af57900ac4eb62b4 is 50, key is test_row_0/B:col10/1731631749492/Put/seqid=0 2024-11-15T00:49:12,141 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111580cd838e9b8b424ba0d60391c255a680_88182fdac0358e5518c99ec7f861a42a store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:12,143 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111580cd838e9b8b424ba0d60391c255a680_88182fdac0358e5518c99ec7f861a42a, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:12,143 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111580cd838e9b8b424ba0d60391c255a680_88182fdac0358e5518c99ec7f861a42a because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:12,146 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742013_1189 (size=13051) 2024-11-15T00:49:12,159 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/c75fb14d546e45a3af57900ac4eb62b4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/c75fb14d546e45a3af57900ac4eb62b4 2024-11-15T00:49:12,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742014_1190 (size=4469) 2024-11-15T00:49:12,172 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/B of 88182fdac0358e5518c99ec7f861a42a into c75fb14d546e45a3af57900ac4eb62b4(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:12,172 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:12,172 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/B, priority=13, startTime=1731631752098; duration=0sec 2024-11-15T00:49:12,172 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:12,172 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:B 2024-11-15T00:49:12,172 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:12,174 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#A#compaction#151 average throughput is 0.44 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:12,174 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:12,174 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 88182fdac0358e5518c99ec7f861a42a/C is initiating minor compaction (all files) 2024-11-15T00:49:12,174 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 88182fdac0358e5518c99ec7f861a42a/C in TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:12,174 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/962b1ec1d5c74526a629a0bdbcfcf7c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/ba95d647d69943f9aac3e7a790bf4280, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6ea6859292cc405e9904c8dc47771f7f] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp, totalSize=36.7 K 2024-11-15T00:49:12,174 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/25746f2c0050408b96c88812dcddf543 is 175, key is test_row_0/A:col10/1731631749492/Put/seqid=0 2024-11-15T00:49:12,174 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 962b1ec1d5c74526a629a0bdbcfcf7c0, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=273, earliestPutTs=1731631746085 2024-11-15T00:49:12,175 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ba95d647d69943f9aac3e7a790bf4280, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1731631747277 2024-11-15T00:49:12,175 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ea6859292cc405e9904c8dc47771f7f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1731631749492 2024-11-15T00:49:12,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742015_1191 (size=32005) 2024-11-15T00:49:12,190 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 88182fdac0358e5518c99ec7f861a42a#C#compaction#153 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:12,191 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/7b3ae9cb59214de5bac2ed321cea237b is 50, key is test_row_0/C:col10/1731631749492/Put/seqid=0 2024-11-15T00:49:12,207 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/25746f2c0050408b96c88812dcddf543 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/25746f2c0050408b96c88812dcddf543 2024-11-15T00:49:12,215 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/A of 88182fdac0358e5518c99ec7f861a42a into 25746f2c0050408b96c88812dcddf543(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:12,216 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:12,216 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/A, priority=13, startTime=1731631752098; duration=0sec 2024-11-15T00:49:12,216 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:12,216 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:A 2024-11-15T00:49:12,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742016_1192 (size=13051) 2024-11-15T00:49:12,226 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/7b3ae9cb59214de5bac2ed321cea237b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7b3ae9cb59214de5bac2ed321cea237b 2024-11-15T00:49:12,245 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 88182fdac0358e5518c99ec7f861a42a/C of 88182fdac0358e5518c99ec7f861a42a into 7b3ae9cb59214de5bac2ed321cea237b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:12,245 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:12,245 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., storeName=88182fdac0358e5518c99ec7f861a42a/C, priority=13, startTime=1731631752098; duration=0sec 2024-11-15T00:49:12,245 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:12,245 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 88182fdac0358e5518c99ec7f861a42a:C 2024-11-15T00:49:12,251 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-15T00:49:12,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:12,252 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T00:49:12,263 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115249224f4665d4df0b9bdb76716998a14_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631750668/Put/seqid=0 2024-11-15T00:49:12,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742017_1193 (size=12454) 2024-11-15T00:49:12,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,284 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115249224f4665d4df0b9bdb76716998a14_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115249224f4665d4df0b9bdb76716998a14_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:12,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/c70c988406e542ddb45c6539fe447806, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:12,288 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/c70c988406e542ddb45c6539fe447806 is 175, key is test_row_0/A:col10/1731631750668/Put/seqid=0 2024-11-15T00:49:12,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742018_1194 (size=31255) 2024-11-15T00:49:12,298 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=333, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/c70c988406e542ddb45c6539fe447806 2024-11-15T00:49:12,309 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/06e6b48c9780410c933653fcce387f24 is 50, key is test_row_0/B:col10/1731631750668/Put/seqid=0 2024-11-15T00:49:12,323 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742019_1195 (size=12301) 2024-11-15T00:49:12,328 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/06e6b48c9780410c933653fcce387f24 2024-11-15T00:49:12,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/9601dfc5cdaf45dba06ca66bff30b903 is 50, key is test_row_0/C:col10/1731631750668/Put/seqid=0 2024-11-15T00:49:12,355 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742020_1196 (size=12301) 2024-11-15T00:49:12,357 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=333 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/9601dfc5cdaf45dba06ca66bff30b903 2024-11-15T00:49:12,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/c70c988406e542ddb45c6539fe447806 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/c70c988406e542ddb45c6539fe447806 2024-11-15T00:49:12,371 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/c70c988406e542ddb45c6539fe447806, entries=150, sequenceid=333, filesize=30.5 K 2024-11-15T00:49:12,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/06e6b48c9780410c933653fcce387f24 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/06e6b48c9780410c933653fcce387f24 2024-11-15T00:49:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,377 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/06e6b48c9780410c933653fcce387f24, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T00:49:12,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/9601dfc5cdaf45dba06ca66bff30b903 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/9601dfc5cdaf45dba06ca66bff30b903 2024-11-15T00:49:12,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,386 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/9601dfc5cdaf45dba06ca66bff30b903, entries=150, sequenceid=333, filesize=12.0 K 2024-11-15T00:49:12,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,387 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=0 B/0 for 88182fdac0358e5518c99ec7f861a42a in 136ms, sequenceid=333, compaction requested=false 2024-11-15T00:49:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:12,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-15T00:49:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-15T00:49:12,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-15T00:49:12,392 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 600 msec 2024-11-15T00:49:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,394 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 606 msec 2024-11-15T00:49:12,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-15T00:49:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,417 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:12,417 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-15T00:49:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,419 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T00:49:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,420 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:12,420 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T00:49:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,532 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,533 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-15T00:49:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:12,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 88182fdac0358e5518c99ec7f861a42a: 2024-11-15T00:49:12,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:12,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-15T00:49:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-15T00:49:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-15T00:49:12,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,580 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 156 msec 2024-11-15T00:49:12,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,584 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 164 msec 2024-11-15T00:49:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-15T00:49:12,737 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,756 INFO [Thread-539 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-539. 2024-11-15T00:49:12,756 DEBUG [Thread-539 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:12,756 DEBUG [Thread-539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,756 DEBUG [Thread-539 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,756 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:12,757 INFO [Thread-541 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-541. 2024-11-15T00:49:12,757 DEBUG [Thread-541 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:12,757 DEBUG [Thread-541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,757 DEBUG [Thread-541 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,757 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:12,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,776 INFO [Thread-545 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-545. 2024-11-15T00:49:12,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,776 DEBUG [Thread-545 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:12,776 DEBUG [Thread-545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,776 DEBUG [Thread-545 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,776 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:12,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,828 INFO [Thread-532 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-532. 2024-11-15T00:49:12,828 DEBUG [Thread-532 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:12,828 DEBUG [Thread-532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,828 DEBUG [Thread-532 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,828 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:12,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,829 INFO [Thread-536 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-536. 2024-11-15T00:49:12,829 DEBUG [Thread-536 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:12,829 DEBUG [Thread-536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,829 DEBUG [Thread-536 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,829 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:12,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:12,838 INFO [Thread-543 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-543. 2024-11-15T00:49:12,838 DEBUG [Thread-543 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:12,838 DEBUG [Thread-543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,838 DEBUG [Thread-543 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,838 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:12,839 INFO [Thread-530 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-530. 2024-11-15T00:49:12,839 INFO [Thread-528 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-528. 2024-11-15T00:49:12,839 INFO [Thread-534 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-534. 2024-11-15T00:49:12,839 DEBUG [Thread-528 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:12,839 DEBUG [Thread-530 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:12,839 DEBUG [Thread-534 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:12,839 DEBUG [Thread-528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,839 DEBUG [Thread-530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,839 DEBUG [Thread-534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,839 DEBUG [Thread-530 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,839 DEBUG [Thread-534 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,839 DEBUG [Thread-528 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-15T00:49:12,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:12,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 41 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 117 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2875 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2832 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1317 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3950 rows 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1318 2024-11-15T00:49:12,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3954 rows 2024-11-15T00:49:12,840 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:49:12,840 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:49:12,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,840 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:12,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:12,841 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:49:12,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:12,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:49:12,843 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631752843"}]},"ts":"1731631752843"} 2024-11-15T00:49:12,845 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:49:12,845 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:49:12,845 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:49:12,847 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, UNASSIGN}] 2024-11-15T00:49:12,847 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, UNASSIGN 2024-11-15T00:49:12,848 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=88182fdac0358e5518c99ec7f861a42a, regionState=CLOSING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:12,849 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=60, ppid=59, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, UNASSIGN because future has completed 2024-11-15T00:49:12,849 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:49:12,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:49:12,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:49:13,002 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(122): Close 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:13,002 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:49:13,002 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1722): Closing 88182fdac0358e5518c99ec7f861a42a, disabling compactions & flushes 2024-11-15T00:49:13,002 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:13,002 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:13,002 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. after waiting 0 ms 2024-11-15T00:49:13,002 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:13,002 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(2902): Flushing 88182fdac0358e5518c99ec7f861a42a 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T00:49:13,011 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115be0250e179a641568133942481cb3f77_88182fdac0358e5518c99ec7f861a42a is 50, key is test_row_0/A:col10/1731631752827/Put/seqid=0 2024-11-15T00:49:13,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742021_1197 (size=12454) 2024-11-15T00:49:13,021 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:13,029 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115be0250e179a641568133942481cb3f77_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115be0250e179a641568133942481cb3f77_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:13,030 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/05f0c5b0feaa4cb8a3e5019dca98e530, store: [table=TestAcidGuarantees family=A region=88182fdac0358e5518c99ec7f861a42a] 2024-11-15T00:49:13,030 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/05f0c5b0feaa4cb8a3e5019dca98e530 is 175, key is test_row_0/A:col10/1731631752827/Put/seqid=0 2024-11-15T00:49:13,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742022_1198 (size=31255) 2024-11-15T00:49:13,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:49:13,437 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=342, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/05f0c5b0feaa4cb8a3e5019dca98e530 2024-11-15T00:49:13,444 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/11dd6ce2c83643639655abc5aa5e6f69 is 50, key is test_row_0/B:col10/1731631752827/Put/seqid=0 2024-11-15T00:49:13,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742023_1199 (size=12301) 2024-11-15T00:49:13,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:49:13,849 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/11dd6ce2c83643639655abc5aa5e6f69 2024-11-15T00:49:13,856 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/a2f023a0c18e40bb9bbbe2e81e40a1d1 is 50, key is test_row_0/C:col10/1731631752827/Put/seqid=0 2024-11-15T00:49:13,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742024_1200 (size=12301) 2024-11-15T00:49:13,861 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=342 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/a2f023a0c18e40bb9bbbe2e81e40a1d1 2024-11-15T00:49:13,865 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/A/05f0c5b0feaa4cb8a3e5019dca98e530 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/05f0c5b0feaa4cb8a3e5019dca98e530 2024-11-15T00:49:13,869 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/05f0c5b0feaa4cb8a3e5019dca98e530, entries=150, sequenceid=342, filesize=30.5 K 2024-11-15T00:49:13,870 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/B/11dd6ce2c83643639655abc5aa5e6f69 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/11dd6ce2c83643639655abc5aa5e6f69 2024-11-15T00:49:13,876 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/11dd6ce2c83643639655abc5aa5e6f69, entries=150, sequenceid=342, filesize=12.0 K 2024-11-15T00:49:13,877 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/.tmp/C/a2f023a0c18e40bb9bbbe2e81e40a1d1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/a2f023a0c18e40bb9bbbe2e81e40a1d1 2024-11-15T00:49:13,881 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/a2f023a0c18e40bb9bbbe2e81e40a1d1, entries=150, sequenceid=342, filesize=12.0 K 2024-11-15T00:49:13,882 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 88182fdac0358e5518c99ec7f861a42a in 880ms, sequenceid=342, compaction requested=true 2024-11-15T00:49:13,883 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/7b42f1ee8d884ef9ac3f2982699a89c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b103ea7028ad4f8faa9bc1e046a3cc45, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b8636236d07841a2aa3e8ea5f4b07cba, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/10d0c988a7c2481fb2adbec429eeab6f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/20b54b1fb03b4573a625e96f0ca4d7ea, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/54cbdc6fe7f449038158df06fdee81f4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/dd45d546a93444338a3bf6f93184bb1a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ccf3a87dab0f4f3d8825ffc18c3cb30b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ff7db022d1da4a988dfc63b0c4c21665, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ceca7c51472f4796b5057a7cc23779cd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2a10f258f8954079a58c2215951bbdc6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/a5203fbe096f4f8fa886172131be6383, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/0d40246c45884a0785efc40231b56363, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ea3cde91d30b4ea7869a3952ebc25eae, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/798ef5f7609d497c9a2135f0fbf9770a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2f3cfc9f15b447e6bcae007a4fffb40b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/6da95f8967f44298becb51840a5f27d9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/f736e0f99b154e75ac635837a4c992c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/185a794ee9214dcb951227fb65755f79, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/d53954d2e24948d980d5d8b78e031d8a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/5b337921b9cf436b9e241f036ac7354e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/1644cf9b1c8c48d5956dd862a8d136b1] to archive 2024-11-15T00:49:13,884 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:49:13,885 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/7b42f1ee8d884ef9ac3f2982699a89c0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/7b42f1ee8d884ef9ac3f2982699a89c0 2024-11-15T00:49:13,886 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b103ea7028ad4f8faa9bc1e046a3cc45 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b103ea7028ad4f8faa9bc1e046a3cc45 2024-11-15T00:49:13,887 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b8636236d07841a2aa3e8ea5f4b07cba to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/b8636236d07841a2aa3e8ea5f4b07cba 2024-11-15T00:49:13,888 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/10d0c988a7c2481fb2adbec429eeab6f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/10d0c988a7c2481fb2adbec429eeab6f 2024-11-15T00:49:13,889 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/20b54b1fb03b4573a625e96f0ca4d7ea to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/20b54b1fb03b4573a625e96f0ca4d7ea 2024-11-15T00:49:13,891 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/54cbdc6fe7f449038158df06fdee81f4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/54cbdc6fe7f449038158df06fdee81f4 2024-11-15T00:49:13,893 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/dd45d546a93444338a3bf6f93184bb1a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/dd45d546a93444338a3bf6f93184bb1a 2024-11-15T00:49:13,895 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ccf3a87dab0f4f3d8825ffc18c3cb30b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ccf3a87dab0f4f3d8825ffc18c3cb30b 2024-11-15T00:49:13,902 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ff7db022d1da4a988dfc63b0c4c21665 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ff7db022d1da4a988dfc63b0c4c21665 2024-11-15T00:49:13,904 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ceca7c51472f4796b5057a7cc23779cd to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ceca7c51472f4796b5057a7cc23779cd 2024-11-15T00:49:13,905 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2a10f258f8954079a58c2215951bbdc6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2a10f258f8954079a58c2215951bbdc6 2024-11-15T00:49:13,907 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/a5203fbe096f4f8fa886172131be6383 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/a5203fbe096f4f8fa886172131be6383 2024-11-15T00:49:13,908 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/0d40246c45884a0785efc40231b56363 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/0d40246c45884a0785efc40231b56363 2024-11-15T00:49:13,909 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ea3cde91d30b4ea7869a3952ebc25eae to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/ea3cde91d30b4ea7869a3952ebc25eae 2024-11-15T00:49:13,911 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/798ef5f7609d497c9a2135f0fbf9770a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/798ef5f7609d497c9a2135f0fbf9770a 2024-11-15T00:49:13,913 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2f3cfc9f15b447e6bcae007a4fffb40b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/2f3cfc9f15b447e6bcae007a4fffb40b 2024-11-15T00:49:13,914 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/6da95f8967f44298becb51840a5f27d9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/6da95f8967f44298becb51840a5f27d9 2024-11-15T00:49:13,915 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/f736e0f99b154e75ac635837a4c992c0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/f736e0f99b154e75ac635837a4c992c0 2024-11-15T00:49:13,916 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/185a794ee9214dcb951227fb65755f79 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/185a794ee9214dcb951227fb65755f79 2024-11-15T00:49:13,918 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/d53954d2e24948d980d5d8b78e031d8a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/d53954d2e24948d980d5d8b78e031d8a 2024-11-15T00:49:13,919 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/5b337921b9cf436b9e241f036ac7354e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/5b337921b9cf436b9e241f036ac7354e 2024-11-15T00:49:13,920 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/1644cf9b1c8c48d5956dd862a8d136b1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/1644cf9b1c8c48d5956dd862a8d136b1 2024-11-15T00:49:13,924 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/c0d9b8eaba614c3bab7dfb28526db7f5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/db409ca7a39844d38aeb774ca156113c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/5a8501dd3b1f4094af2e31d1cac773c4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8501394c900e4f8fac136a8693ddbf7e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e2888002e68f4caea4d5164886c37546, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8431745a633541bd8a19dfa9654135a2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/fe64fed092994467b8e4d5e076808365, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9d892d81f1bc4f9aac55e9fdc1814651, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/0b2c2e83445f43c880715f801aa9c556, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/3c123e4cc7ad4debb4c3f286331f6ece, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/90875d72c16b4e019599ae5b53dc3f2a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/5f4221248f3843d29f2c88e28e5d4678, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/76c36715972e4d8185356418595fe1f4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9f358eb6721f4665818c15454810330c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/995a6a196dab4a0296bd54511bcc5d78, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/681c89297fc94910ac7dccb878121578, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/02ba5f1f92f343278161b67b4dc8bca6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/7d584fef372e4f84b2b7e858b329abe4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/66d9a9ed2cc0491ab98e9301883027c7, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b3f9c258d9934a43a30c7dc3303cc96a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e5f420acb2c94753a468ebbb0b46549c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b48ff679700744688a546a2b5ec19e5a] to archive 2024-11-15T00:49:13,927 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:49:13,929 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/c0d9b8eaba614c3bab7dfb28526db7f5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/c0d9b8eaba614c3bab7dfb28526db7f5 2024-11-15T00:49:13,931 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/db409ca7a39844d38aeb774ca156113c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/db409ca7a39844d38aeb774ca156113c 2024-11-15T00:49:13,932 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/5a8501dd3b1f4094af2e31d1cac773c4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/5a8501dd3b1f4094af2e31d1cac773c4 2024-11-15T00:49:13,934 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8501394c900e4f8fac136a8693ddbf7e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8501394c900e4f8fac136a8693ddbf7e 2024-11-15T00:49:13,935 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e2888002e68f4caea4d5164886c37546 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e2888002e68f4caea4d5164886c37546 2024-11-15T00:49:13,937 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8431745a633541bd8a19dfa9654135a2 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/8431745a633541bd8a19dfa9654135a2 2024-11-15T00:49:13,939 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/fe64fed092994467b8e4d5e076808365 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/fe64fed092994467b8e4d5e076808365 2024-11-15T00:49:13,941 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9d892d81f1bc4f9aac55e9fdc1814651 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9d892d81f1bc4f9aac55e9fdc1814651 2024-11-15T00:49:13,942 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/0b2c2e83445f43c880715f801aa9c556 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/0b2c2e83445f43c880715f801aa9c556 2024-11-15T00:49:13,944 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/3c123e4cc7ad4debb4c3f286331f6ece to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/3c123e4cc7ad4debb4c3f286331f6ece 2024-11-15T00:49:13,946 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/90875d72c16b4e019599ae5b53dc3f2a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/90875d72c16b4e019599ae5b53dc3f2a 2024-11-15T00:49:13,947 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/5f4221248f3843d29f2c88e28e5d4678 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/5f4221248f3843d29f2c88e28e5d4678 2024-11-15T00:49:13,948 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/76c36715972e4d8185356418595fe1f4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/76c36715972e4d8185356418595fe1f4 2024-11-15T00:49:13,949 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9f358eb6721f4665818c15454810330c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/9f358eb6721f4665818c15454810330c 2024-11-15T00:49:13,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/995a6a196dab4a0296bd54511bcc5d78 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/995a6a196dab4a0296bd54511bcc5d78 2024-11-15T00:49:13,950 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/681c89297fc94910ac7dccb878121578 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/681c89297fc94910ac7dccb878121578 2024-11-15T00:49:13,951 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/02ba5f1f92f343278161b67b4dc8bca6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/02ba5f1f92f343278161b67b4dc8bca6 2024-11-15T00:49:13,953 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/7d584fef372e4f84b2b7e858b329abe4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/7d584fef372e4f84b2b7e858b329abe4 2024-11-15T00:49:13,954 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/66d9a9ed2cc0491ab98e9301883027c7 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/66d9a9ed2cc0491ab98e9301883027c7 2024-11-15T00:49:13,955 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b3f9c258d9934a43a30c7dc3303cc96a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b3f9c258d9934a43a30c7dc3303cc96a 2024-11-15T00:49:13,956 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e5f420acb2c94753a468ebbb0b46549c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/e5f420acb2c94753a468ebbb0b46549c 2024-11-15T00:49:13,957 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b48ff679700744688a546a2b5ec19e5a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/b48ff679700744688a546a2b5ec19e5a 2024-11-15T00:49:13,958 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/0f9cbb908d5641f3a3a02cc057a4ab64, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/463fee77addc40bc93d42e4f27d9a742, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/78c754d421854bb6bd31e59c5141bb17, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6066bd86a95048afb125c147fa644eaf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/87ab52a69262448691b87d37ac9f651f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/cdfd346e9ad349d4a82293680a1305b0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/25e00f2d466d4fbebad49d0882d54cc1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/3d63a125949f4ddda74a3b7f2d4b62e3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/cd401035fa3e437a846874db1db34a85, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/5e76eff5ab7f429cb715d080b40d2d21, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/df8ae0c721e14401839486206c8c2c4e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/8dc14e7d7d864cbfa050faabb16cf44c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7f3f4c14990142fc9807fbf09cf6fd78, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/008bd2d0f9d04eb2a737e220a5985010, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/36d114f6f7f5432187c04bff583bb5fa, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7778e5e326f64ec69fd3202d6be62e45, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/fad6578f0b1349c387ba3d0e6a875dc3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/1b3b3c2f6089478fad7d8988c5f9769e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/962b1ec1d5c74526a629a0bdbcfcf7c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/39952c0024044b66a94fe262629bc983, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/ba95d647d69943f9aac3e7a790bf4280, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6ea6859292cc405e9904c8dc47771f7f] to archive 2024-11-15T00:49:13,959 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:49:13,961 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/0f9cbb908d5641f3a3a02cc057a4ab64 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/0f9cbb908d5641f3a3a02cc057a4ab64 2024-11-15T00:49:13,963 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/463fee77addc40bc93d42e4f27d9a742 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/463fee77addc40bc93d42e4f27d9a742 2024-11-15T00:49:13,964 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/78c754d421854bb6bd31e59c5141bb17 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/78c754d421854bb6bd31e59c5141bb17 2024-11-15T00:49:13,965 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6066bd86a95048afb125c147fa644eaf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6066bd86a95048afb125c147fa644eaf 2024-11-15T00:49:13,966 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/87ab52a69262448691b87d37ac9f651f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/87ab52a69262448691b87d37ac9f651f 2024-11-15T00:49:13,967 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/cdfd346e9ad349d4a82293680a1305b0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/cdfd346e9ad349d4a82293680a1305b0 2024-11-15T00:49:13,968 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/25e00f2d466d4fbebad49d0882d54cc1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/25e00f2d466d4fbebad49d0882d54cc1 2024-11-15T00:49:13,969 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/3d63a125949f4ddda74a3b7f2d4b62e3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/3d63a125949f4ddda74a3b7f2d4b62e3 2024-11-15T00:49:13,971 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/cd401035fa3e437a846874db1db34a85 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/cd401035fa3e437a846874db1db34a85 2024-11-15T00:49:13,973 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/5e76eff5ab7f429cb715d080b40d2d21 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/5e76eff5ab7f429cb715d080b40d2d21 2024-11-15T00:49:13,975 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/df8ae0c721e14401839486206c8c2c4e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/df8ae0c721e14401839486206c8c2c4e 2024-11-15T00:49:13,976 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/8dc14e7d7d864cbfa050faabb16cf44c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/8dc14e7d7d864cbfa050faabb16cf44c 2024-11-15T00:49:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:49:13,977 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7f3f4c14990142fc9807fbf09cf6fd78 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7f3f4c14990142fc9807fbf09cf6fd78 2024-11-15T00:49:13,978 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/008bd2d0f9d04eb2a737e220a5985010 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/008bd2d0f9d04eb2a737e220a5985010 2024-11-15T00:49:13,979 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/36d114f6f7f5432187c04bff583bb5fa to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/36d114f6f7f5432187c04bff583bb5fa 2024-11-15T00:49:13,981 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7778e5e326f64ec69fd3202d6be62e45 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7778e5e326f64ec69fd3202d6be62e45 2024-11-15T00:49:13,982 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/fad6578f0b1349c387ba3d0e6a875dc3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/fad6578f0b1349c387ba3d0e6a875dc3 2024-11-15T00:49:13,983 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/1b3b3c2f6089478fad7d8988c5f9769e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/1b3b3c2f6089478fad7d8988c5f9769e 2024-11-15T00:49:13,984 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/962b1ec1d5c74526a629a0bdbcfcf7c0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/962b1ec1d5c74526a629a0bdbcfcf7c0 2024-11-15T00:49:13,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/39952c0024044b66a94fe262629bc983 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/39952c0024044b66a94fe262629bc983 2024-11-15T00:49:13,989 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/ba95d647d69943f9aac3e7a790bf4280 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/ba95d647d69943f9aac3e7a790bf4280 2024-11-15T00:49:13,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6ea6859292cc405e9904c8dc47771f7f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/6ea6859292cc405e9904c8dc47771f7f 2024-11-15T00:49:13,998 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/recovered.edits/345.seqid, newMaxSeqId=345, maxSeqId=4 2024-11-15T00:49:13,999 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. 2024-11-15T00:49:13,999 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] regionserver.HRegion(1676): Region close journal for 88182fdac0358e5518c99ec7f861a42a: Waiting for close lock at 1731631753002Running coprocessor pre-close hooks at 1731631753002Disabling compacts and flushes for region at 1731631753002Disabling writes for close at 1731631753002Obtaining lock to block concurrent updates at 1731631753002Preparing flush snapshotting stores in 88182fdac0358e5518c99ec7f861a42a at 1731631753002Finished memstore snapshotting TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731631753003 (+1 ms)Flushing stores of TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a. at 1731631753004 (+1 ms)Flushing 88182fdac0358e5518c99ec7f861a42a/A: creating writer at 1731631753004Flushing mob file 88182fdac0358e5518c99ec7f861a42a/A: appending metadata at 1731631753011 (+7 ms)Flushing mob file 88182fdac0358e5518c99ec7f861a42a/A: closing flushed file at 1731631753011Flushing 88182fdac0358e5518c99ec7f861a42a/A: appending metadata at 1731631753030 (+19 ms)Flushing 88182fdac0358e5518c99ec7f861a42a/A: closing flushed file at 1731631753030Flushing 88182fdac0358e5518c99ec7f861a42a/B: creating writer at 1731631753442 (+412 ms)Flushing 88182fdac0358e5518c99ec7f861a42a/B: appending metadata at 1731631753444 (+2 ms)Flushing 88182fdac0358e5518c99ec7f861a42a/B: closing flushed file at 1731631753444Flushing 88182fdac0358e5518c99ec7f861a42a/C: creating writer at 1731631753853 (+409 ms)Flushing 88182fdac0358e5518c99ec7f861a42a/C: appending metadata at 1731631753855 (+2 ms)Flushing 88182fdac0358e5518c99ec7f861a42a/C: closing flushed file at 1731631753855Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ea77276: reopening flushed file at 1731631753864 (+9 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4cf1fcc6: reopening flushed file at 1731631753869 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@59b09eb8: reopening flushed file at 1731631753877 (+8 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 88182fdac0358e5518c99ec7f861a42a in 880ms, sequenceid=342, compaction requested=true at 1731631753882 (+5 ms)Writing region close event to WAL at 1731631753993 (+111 ms)Running coprocessor post-close hooks at 1731631753998 (+5 ms)Closed at 1731631753999 (+1 ms) 2024-11-15T00:49:14,002 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=61}] handler.UnassignRegionHandler(157): Closed 88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:14,005 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=60 updating hbase:meta row=88182fdac0358e5518c99ec7f861a42a, regionState=CLOSED 2024-11-15T00:49:14,007 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=61, ppid=60, state=RUNNABLE, hasLock=false; CloseRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:49:14,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-15T00:49:14,012 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; CloseRegionProcedure 88182fdac0358e5518c99ec7f861a42a, server=fb2f6cb7846a,42925,1731631698709 in 1.1600 sec 2024-11-15T00:49:14,014 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=60, resume processing ppid=59 2024-11-15T00:49:14,015 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, ppid=59, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=88182fdac0358e5518c99ec7f861a42a, UNASSIGN in 1.1650 sec 2024-11-15T00:49:14,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-15T00:49:14,017 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.1700 sec 2024-11-15T00:49:14,019 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631754019"}]},"ts":"1731631754019"} 2024-11-15T00:49:14,020 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:49:14,020 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:49:14,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-15T00:49:14,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-15T00:49:14,987 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:14,988 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:49:14,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:14,989 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=62, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:14,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T00:49:14,990 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=62, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:14,996 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:14,999 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/recovered.edits] 2024-11-15T00:49:15,001 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/05f0c5b0feaa4cb8a3e5019dca98e530 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/05f0c5b0feaa4cb8a3e5019dca98e530 2024-11-15T00:49:15,003 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/25746f2c0050408b96c88812dcddf543 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/25746f2c0050408b96c88812dcddf543 2024-11-15T00:49:15,004 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/c70c988406e542ddb45c6539fe447806 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/A/c70c988406e542ddb45c6539fe447806 2024-11-15T00:49:15,006 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/06e6b48c9780410c933653fcce387f24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/06e6b48c9780410c933653fcce387f24 2024-11-15T00:49:15,007 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/11dd6ce2c83643639655abc5aa5e6f69 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/11dd6ce2c83643639655abc5aa5e6f69 2024-11-15T00:49:15,008 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/c75fb14d546e45a3af57900ac4eb62b4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/B/c75fb14d546e45a3af57900ac4eb62b4 2024-11-15T00:49:15,011 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7b3ae9cb59214de5bac2ed321cea237b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/7b3ae9cb59214de5bac2ed321cea237b 2024-11-15T00:49:15,013 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/9601dfc5cdaf45dba06ca66bff30b903 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/9601dfc5cdaf45dba06ca66bff30b903 2024-11-15T00:49:15,015 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/a2f023a0c18e40bb9bbbe2e81e40a1d1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/C/a2f023a0c18e40bb9bbbe2e81e40a1d1 2024-11-15T00:49:15,018 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/recovered.edits/345.seqid to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a/recovered.edits/345.seqid 2024-11-15T00:49:15,018 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,019 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:49:15,019 DEBUG [PEWorker-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:49:15,020 DEBUG [PEWorker-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T00:49:15,024 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115068aceed0f714a179a94afc8c617c83d_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115068aceed0f714a179a94afc8c617c83d_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,025 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d9e2acfbbd441b0a17cc2d97554d5fa_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411150d9e2acfbbd441b0a17cc2d97554d5fa_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,026 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111513d4efb5008f43c1aa5f2cd916af804c_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111513d4efb5008f43c1aa5f2cd916af804c_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,027 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111514dd7d00142344a8b4dc9a3ee1534eb2_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111514dd7d00142344a8b4dc9a3ee1534eb2_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,029 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111517326d5c7e1b4a26ac6c6413fcfac1af_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111517326d5c7e1b4a26ac6c6413fcfac1af_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,030 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115249224f4665d4df0b9bdb76716998a14_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115249224f4665d4df0b9bdb76716998a14_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,031 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a6d8acdc1234765ad35c55f9194c870_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a6d8acdc1234765ad35c55f9194c870_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,033 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f05113efe7d4075a27b2a9d5ebdff44_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153f05113efe7d4075a27b2a9d5ebdff44_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,034 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111544830d7445f741abb18d9c02ff8e9b12_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111544830d7445f741abb18d9c02ff8e9b12_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,035 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111549fafbdbc0f4450e8157591283546062_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111549fafbdbc0f4450e8157591283546062_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,036 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155e2e9ac82fef41bda3c64b5cc333e81f_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155e2e9ac82fef41bda3c64b5cc333e81f_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,037 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c48b0dd29f042209a74382c19f2e9af_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411157c48b0dd29f042209a74382c19f2e9af_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,038 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a2a337351d9942e7a4981991d4125c91_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115a2a337351d9942e7a4981991d4125c91_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,039 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115be0250e179a641568133942481cb3f77_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115be0250e179a641568133942481cb3f77_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,040 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf4ab97b5bc24ad6971d358b8a16c0f8_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf4ab97b5bc24ad6971d358b8a16c0f8_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,041 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d99b610e18f646e0a8e5393ebf4e4166_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115d99b610e18f646e0a8e5393ebf4e4166_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,042 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ddb2e018e079432d8b4af951451f3b17_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ddb2e018e079432d8b4af951451f3b17_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,043 DEBUG [PEWorker-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f56f89ca9b874f2f943dff9b41a81dd1_88182fdac0358e5518c99ec7f861a42a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f56f89ca9b874f2f943dff9b41a81dd1_88182fdac0358e5518c99ec7f861a42a 2024-11-15T00:49:15,044 DEBUG [PEWorker-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:49:15,046 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=62, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:15,048 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:49:15,050 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:49:15,051 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=62, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:15,051 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:49:15,051 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631755051"}]},"ts":"9223372036854775807"} 2024-11-15T00:49:15,053 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:49:15,053 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 88182fdac0358e5518c99ec7f861a42a, NAME => 'TestAcidGuarantees,,1731631729482.88182fdac0358e5518c99ec7f861a42a.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:49:15,054 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:49:15,054 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631755054"}]},"ts":"9223372036854775807"} 2024-11-15T00:49:15,056 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:49:15,056 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=62, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:15,057 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 68 msec 2024-11-15T00:49:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-15T00:49:15,097 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:49:15,099 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:15,106 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=244 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1613740833_22 at /127.0.0.1:47704 [Waiting for operation #375] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1256560221_22 at /127.0.0.1:44338 [Waiting for operation #740] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/cluster_4d500b26-95f0-bc79-8ccc-862558111e78/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-1256560221_22 at /127.0.0.1:40202 [Waiting for operation #997] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/cluster_4d500b26-95f0-bc79-8ccc-862558111e78/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1613740833_22 at /127.0.0.1:47688 [Waiting for operation #373] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 453) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=896 (was 995), ProcessCount=11 (was 11), AvailableMemoryMB=1854 (was 1977) 2024-11-15T00:49:15,114 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=244, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=896, ProcessCount=11, AvailableMemoryMB=1854 2024-11-15T00:49:15,115 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:49:15,116 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:49:15,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:15,118 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:49:15,118 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:15,118 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 63 2024-11-15T00:49:15,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T00:49:15,119 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:49:15,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742025_1201 (size=959) 2024-11-15T00:49:15,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T00:49:15,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T00:49:15,526 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd 2024-11-15T00:49:15,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742026_1202 (size=53) 2024-11-15T00:49:15,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T00:49:15,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:49:15,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 37711fc54bf4a3c70220e87c7201b430, disabling compactions & flushes 2024-11-15T00:49:15,932 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:15,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:15,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. after waiting 0 ms 2024-11-15T00:49:15,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:15,932 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:15,932 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 37711fc54bf4a3c70220e87c7201b430: Waiting for close lock at 1731631755932Disabling compacts and flushes for region at 1731631755932Disabling writes for close at 1731631755932Writing region close event to WAL at 1731631755932Closed at 1731631755932 2024-11-15T00:49:15,933 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:49:15,934 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631755934"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631755934"}]},"ts":"1731631755934"} 2024-11-15T00:49:15,935 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:49:15,936 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:49:15,936 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631755936"}]},"ts":"1731631755936"} 2024-11-15T00:49:15,937 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:49:15,937 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37711fc54bf4a3c70220e87c7201b430, ASSIGN}] 2024-11-15T00:49:15,938 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37711fc54bf4a3c70220e87c7201b430, ASSIGN 2024-11-15T00:49:15,939 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37711fc54bf4a3c70220e87c7201b430, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,42925,1731631698709; forceNewPlan=false, retain=false 2024-11-15T00:49:16,090 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=37711fc54bf4a3c70220e87c7201b430, regionState=OPENING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:16,091 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37711fc54bf4a3c70220e87c7201b430, ASSIGN because future has completed 2024-11-15T00:49:16,092 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:49:16,246 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:16,246 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7752): Opening region: {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:49:16,247 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,247 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:49:16,247 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7794): checking encryption for 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,247 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(7797): checking classloading for 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,248 INFO [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,250 INFO [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37711fc54bf4a3c70220e87c7201b430 columnFamilyName A 2024-11-15T00:49:16,250 DEBUG [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:16,250 INFO [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] regionserver.HStore(327): Store=37711fc54bf4a3c70220e87c7201b430/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:49:16,250 INFO [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,252 INFO [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37711fc54bf4a3c70220e87c7201b430 columnFamilyName B 2024-11-15T00:49:16,252 DEBUG [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:16,252 INFO [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] regionserver.HStore(327): Store=37711fc54bf4a3c70220e87c7201b430/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:49:16,252 INFO [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,253 INFO [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 37711fc54bf4a3c70220e87c7201b430 columnFamilyName C 2024-11-15T00:49:16,253 DEBUG [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:16,254 INFO [StoreOpener-37711fc54bf4a3c70220e87c7201b430-1 {}] regionserver.HStore(327): Store=37711fc54bf4a3c70220e87c7201b430/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:49:16,254 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1038): replaying wal for 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,254 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,255 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,256 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1048): stopping wal replay for 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,256 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1060): Cleaning up temporary data for 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,256 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:49:16,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T00:49:16,258 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1093): writing seq id for 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,260 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:49:16,260 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1114): Opened 37711fc54bf4a3c70220e87c7201b430; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74758928, jitterRate=0.11399483680725098}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:49:16,260 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:16,261 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegion(1006): Region open journal for 37711fc54bf4a3c70220e87c7201b430: Running coprocessor pre-open hook at 1731631756247Writing region info on filesystem at 1731631756247Initializing all the Stores at 1731631756248 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631756248Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631756248Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631756248Cleaning up temporary data from old regions at 1731631756256 (+8 ms)Running coprocessor post-open hooks at 1731631756260 (+4 ms)Region opened successfully at 1731631756260 2024-11-15T00:49:16,261 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., pid=65, masterSystemTime=1731631756243 2024-11-15T00:49:16,263 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:16,263 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=65}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:16,263 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=37711fc54bf4a3c70220e87c7201b430, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:16,265 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; OpenRegionProcedure 37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:49:16,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-15T00:49:16,267 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; OpenRegionProcedure 37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 in 173 msec 2024-11-15T00:49:16,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-15T00:49:16,268 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37711fc54bf4a3c70220e87c7201b430, ASSIGN in 330 msec 2024-11-15T00:49:16,269 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:49:16,269 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631756269"}]},"ts":"1731631756269"} 2024-11-15T00:49:16,270 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:49:16,271 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=63, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:49:16,273 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1550 sec 2024-11-15T00:49:16,788 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:49:17,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=63 2024-11-15T00:49:17,277 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:17,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1872aec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,278 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,278 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,279 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@73081eea, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,280 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,281 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35678, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c973158, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,281 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,282 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,283 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38522, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,284 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:49:17,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,46397,1731631697729 2024-11-15T00:49:17,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@557c0cae 2024-11-15T00:49:17,285 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:49:17,286 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:49:17,287 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5c863246, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,287 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,287 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,288 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2654a4c5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,288 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,289 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,289 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35712, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,290 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a660aca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,291 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e44b286, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,291 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,291 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,292 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54519fd8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,292 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,293 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35734, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,293 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@598cb0e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,296 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@611ff3c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,296 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,296 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,297 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239d81af, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,297 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,298 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,298 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,299 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@129e841, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,300 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32865d71, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,300 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,300 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,300 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,300 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@af676fa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,301 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,302 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35768, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,302 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@be4a44a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,303 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@67136d23, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,303 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,303 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,304 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c525cf4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,304 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,305 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,305 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35784, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,306 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@b24cef8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,306 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6261277b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,306 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,307 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,307 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,307 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@76c31ad1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,308 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,309 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35802, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@799de632, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,310 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5b50bdc1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,310 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,310 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,311 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d2d9a32, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,311 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,312 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,312 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35808, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@808b3ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,313 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f5f704, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,313 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,313 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,314 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@653e0eeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,314 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,315 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,315 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35840, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,316 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f68f839, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,316 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37ad5c18, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,316 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,316 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,317 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e1db434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,317 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,318 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,318 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35846, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c31a5de, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,319 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d2d949d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,319 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:17,319 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:17,320 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:17,320 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52b358cc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:17,321 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:17,322 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:35866, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:17,322 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6dcdc0e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:17,324 DEBUG [Thread-984 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,324 DEBUG [Thread-965 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,325 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,325 DEBUG [Thread-967 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,325 DEBUG [Thread-971 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,326 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,326 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,326 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38536, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38538, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,327 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38554, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,327 DEBUG [Thread-982 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:49:17,328 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:49:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,329 DEBUG [Thread-978 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:49:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,329 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,330 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees 2024-11-15T00:49:17,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38562, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,331 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38564, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:49:17,332 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38588, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,332 DEBUG [Thread-976 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:49:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:49:17,333 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,334 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:49:17,334 DEBUG [Thread-973 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,335 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,335 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38598, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,336 DEBUG [Thread-969 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,336 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:49:17,337 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,338 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=66, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:17,339 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=67, ppid=66, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:17,339 DEBUG [Thread-969 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,339 DEBUG [Thread-980 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:17,339 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:49:17,341 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38620, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:17,342 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:17,345 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38624, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:17,348 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:49:17,349 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:49:17,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:17,350 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:17,351 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T00:49:17,382 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/327ed577e0f24b498c5f2d1f71e57e7a is 50, key is test_row_0/A:col10/1731631757349/Put/seqid=0 2024-11-15T00:49:17,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742027_1203 (size=12001) 2024-11-15T00:49:17,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631817420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631817420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631817420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,430 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,430 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631817425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,431 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,431 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631817425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,432 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:49:17,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:17,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:17,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:17,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:17,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631817537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,540 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631817537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631817538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,540 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631817538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,541 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631817538, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,541 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,542 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:49:17,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:17,652 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,747 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631817747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,748 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,748 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631817747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631817747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631817747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,749 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,750 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:17,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631817748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:17,751 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:17,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/327ed577e0f24b498c5f2d1f71e57e7a 2024-11-15T00:49:17,805 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:17,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:17,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:17,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:17,805 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,835 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/d902ff7b9e3a489bb2ade4e3d0fd35c1 is 50, key is test_row_0/B:col10/1731631757349/Put/seqid=0 2024-11-15T00:49:17,871 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742028_1204 (size=12001) 2024-11-15T00:49:17,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:49:17,958 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:17,959 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:17,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631818056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,058 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631818057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631818057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,059 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631818057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631818058, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,060 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,112 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,112 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,266 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:18,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:18,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,267 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,272 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/d902ff7b9e3a489bb2ade4e3d0fd35c1 2024-11-15T00:49:18,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/324b54aadb86405c83ed710dd0525fbf is 50, key is test_row_0/C:col10/1731631757349/Put/seqid=0 2024-11-15T00:49:18,342 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742029_1205 (size=12001) 2024-11-15T00:49:18,344 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/324b54aadb86405c83ed710dd0525fbf 2024-11-15T00:49:18,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/327ed577e0f24b498c5f2d1f71e57e7a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/327ed577e0f24b498c5f2d1f71e57e7a 2024-11-15T00:49:18,416 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/327ed577e0f24b498c5f2d1f71e57e7a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T00:49:18,420 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:18,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:18,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,425 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/d902ff7b9e3a489bb2ade4e3d0fd35c1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d902ff7b9e3a489bb2ade4e3d0fd35c1 2024-11-15T00:49:18,436 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d902ff7b9e3a489bb2ade4e3d0fd35c1, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T00:49:18,437 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/324b54aadb86405c83ed710dd0525fbf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/324b54aadb86405c83ed710dd0525fbf 2024-11-15T00:49:18,445 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/324b54aadb86405c83ed710dd0525fbf, entries=150, sequenceid=14, filesize=11.7 K 2024-11-15T00:49:18,446 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 37711fc54bf4a3c70220e87c7201b430 in 1096ms, sequenceid=14, compaction requested=false 2024-11-15T00:49:18,447 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T00:49:18,447 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:18,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:49:18,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:18,578 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:18,578 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:49:18,584 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,585 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/4b82cdd2b11a43f0bf853065fbd2442d is 50, key is test_row_0/A:col10/1731631757394/Put/seqid=0 2024-11-15T00:49:18,641 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742030_1206 (size=12001) 2024-11-15T00:49:18,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631818650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631818650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631818651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,659 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631818659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631818659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,664 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,666 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,741 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:18,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:18,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,742 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,742 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,770 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631818767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,771 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,771 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631818767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631818767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,772 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631818767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,775 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,783 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631818778, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,901 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:18,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:18,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:18,901 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,901 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:18,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631818977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631818977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,982 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631818977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,983 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631818987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,991 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:18,992 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:18,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631818988, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:18,992 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,042 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/4b82cdd2b11a43f0bf853065fbd2442d 2024-11-15T00:49:19,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,118 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/849443a8e80741c89caa2912761d62e0 is 50, key is test_row_0/B:col10/1731631757394/Put/seqid=0 2024-11-15T00:49:19,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742031_1207 (size=12001) 2024-11-15T00:49:19,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,223 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631819287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,289 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631819287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,289 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,291 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:19,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631819287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,291 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:19,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631819297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:19,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631819300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,302 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:19,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:19,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:49:19,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,554 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:49:19,577 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/849443a8e80741c89caa2912761d62e0 2024-11-15T00:49:19,602 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/90a4902ae02b4e5bbd71480c706e8eaa is 50, key is test_row_0/C:col10/1731631757394/Put/seqid=0 2024-11-15T00:49:19,647 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742032_1208 (size=12001) 2024-11-15T00:49:19,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,695 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631819798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:19,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:19,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631819798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,800 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,805 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:19,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631819800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,808 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:19,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631819807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,810 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,820 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:19,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631819817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:19,821 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:19,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:19,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:19,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:20,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:20,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:20,004 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] handler.RSProcedureHandler(58): pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:20,004 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=67 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:20,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=67 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:20,041 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/90a4902ae02b4e5bbd71480c706e8eaa 2024-11-15T00:49:20,051 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/4b82cdd2b11a43f0bf853065fbd2442d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/4b82cdd2b11a43f0bf853065fbd2442d 2024-11-15T00:49:20,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/4b82cdd2b11a43f0bf853065fbd2442d, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T00:49:20,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/849443a8e80741c89caa2912761d62e0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/849443a8e80741c89caa2912761d62e0 2024-11-15T00:49:20,072 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/849443a8e80741c89caa2912761d62e0, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T00:49:20,074 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/90a4902ae02b4e5bbd71480c706e8eaa as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/90a4902ae02b4e5bbd71480c706e8eaa 2024-11-15T00:49:20,097 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/90a4902ae02b4e5bbd71480c706e8eaa, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T00:49:20,098 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=46.96 KB/48090 for 37711fc54bf4a3c70220e87c7201b430 in 1520ms, sequenceid=41, compaction requested=false 2024-11-15T00:49:20,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:20,157 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=67 2024-11-15T00:49:20,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:20,157 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T00:49:20,170 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/442e141812024ed18cd467f78eb91e16 is 50, key is test_row_0/A:col10/1731631758652/Put/seqid=0 2024-11-15T00:49:20,199 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742033_1209 (size=12001) 2024-11-15T00:49:20,199 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/442e141812024ed18cd467f78eb91e16 2024-11-15T00:49:20,236 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/5b08b564d77b498f802147fc22e96e0d is 50, key is test_row_0/B:col10/1731631758652/Put/seqid=0 2024-11-15T00:49:20,281 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742034_1210 (size=12001) 2024-11-15T00:49:20,281 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/5b08b564d77b498f802147fc22e96e0d 2024-11-15T00:49:20,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/4388d8edf4064515a9d3061c74ebc1c3 is 50, key is test_row_0/C:col10/1731631758652/Put/seqid=0 2024-11-15T00:49:20,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742035_1211 (size=12001) 2024-11-15T00:49:20,748 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=51 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/4388d8edf4064515a9d3061c74ebc1c3 2024-11-15T00:49:20,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/442e141812024ed18cd467f78eb91e16 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/442e141812024ed18cd467f78eb91e16 2024-11-15T00:49:20,765 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/442e141812024ed18cd467f78eb91e16, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T00:49:20,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/5b08b564d77b498f802147fc22e96e0d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/5b08b564d77b498f802147fc22e96e0d 2024-11-15T00:49:20,781 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/5b08b564d77b498f802147fc22e96e0d, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T00:49:20,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/4388d8edf4064515a9d3061c74ebc1c3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4388d8edf4064515a9d3061c74ebc1c3 2024-11-15T00:49:20,812 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4388d8edf4064515a9d3061c74ebc1c3, entries=150, sequenceid=51, filesize=11.7 K 2024-11-15T00:49:20,813 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=0 B/0 for 37711fc54bf4a3c70220e87c7201b430 in 656ms, sequenceid=51, compaction requested=true 2024-11-15T00:49:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:20,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=67}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=67 2024-11-15T00:49:20,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=67 2024-11-15T00:49:20,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=67, resume processing ppid=66 2024-11-15T00:49:20,820 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, ppid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.4780 sec 2024-11-15T00:49:20,825 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=66, table=TestAcidGuarantees in 3.4960 sec 2024-11-15T00:49:20,840 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:20,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:49:20,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:20,848 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/3b0813c8f9094ecf9f166dbc48da24c9 is 50, key is test_row_0/A:col10/1731631760826/Put/seqid=0 2024-11-15T00:49:20,890 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742036_1212 (size=14341) 2024-11-15T00:49:20,896 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/3b0813c8f9094ecf9f166dbc48da24c9 2024-11-15T00:49:20,898 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:20,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631820890, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:20,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631820892, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:20,899 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:20,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631820893, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:20,899 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:20,900 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:20,905 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631820898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:20,905 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:20,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631820899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:20,906 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:20,922 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/9691f31b837444d2a2e63bea194f78f6 is 50, key is test_row_0/B:col10/1731631760826/Put/seqid=0 2024-11-15T00:49:21,015 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631821007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,016 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631821007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631821007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,017 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742037_1213 (size=12001) 2024-11-15T00:49:21,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631821017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,023 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/9691f31b837444d2a2e63bea194f78f6 2024-11-15T00:49:21,023 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631821017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,023 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/e2afa2f4f5144fa2b6888535f6fa534c is 50, key is test_row_0/C:col10/1731631760826/Put/seqid=0 2024-11-15T00:49:21,075 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742038_1214 (size=12001) 2024-11-15T00:49:21,235 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631821229, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631821230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631821230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,237 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631821230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,238 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631821230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,238 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,248 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,475 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=63 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/e2afa2f4f5144fa2b6888535f6fa534c 2024-11-15T00:49:21,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/3b0813c8f9094ecf9f166dbc48da24c9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3b0813c8f9094ecf9f166dbc48da24c9 2024-11-15T00:49:21,486 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3b0813c8f9094ecf9f166dbc48da24c9, entries=200, sequenceid=63, filesize=14.0 K 2024-11-15T00:49:21,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-15T00:49:21,487 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:21,487 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/9691f31b837444d2a2e63bea194f78f6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/9691f31b837444d2a2e63bea194f78f6 2024-11-15T00:49:21,487 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:21,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees 2024-11-15T00:49:21,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:49:21,489 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:21,490 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=68, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:21,490 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:21,494 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/9691f31b837444d2a2e63bea194f78f6, entries=150, sequenceid=63, filesize=11.7 K 2024-11-15T00:49:21,495 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/e2afa2f4f5144fa2b6888535f6fa534c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2afa2f4f5144fa2b6888535f6fa534c 2024-11-15T00:49:21,507 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2afa2f4f5144fa2b6888535f6fa534c, entries=150, sequenceid=63, filesize=11.7 K 2024-11-15T00:49:21,508 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 37711fc54bf4a3c70220e87c7201b430 in 668ms, sequenceid=63, compaction requested=true 2024-11-15T00:49:21,508 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:21,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:21,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:21,508 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:21,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:21,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:21,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:21,508 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:49:21,508 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:21,510 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50344 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:21,510 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:21,510 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/A is initiating minor compaction (all files) 2024-11-15T00:49:21,510 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/B is initiating minor compaction (all files) 2024-11-15T00:49:21,510 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/A in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:21,510 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/B in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:21,510 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/327ed577e0f24b498c5f2d1f71e57e7a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/4b82cdd2b11a43f0bf853065fbd2442d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/442e141812024ed18cd467f78eb91e16, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3b0813c8f9094ecf9f166dbc48da24c9] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=49.2 K 2024-11-15T00:49:21,510 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d902ff7b9e3a489bb2ade4e3d0fd35c1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/849443a8e80741c89caa2912761d62e0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/5b08b564d77b498f802147fc22e96e0d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/9691f31b837444d2a2e63bea194f78f6] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=46.9 K 2024-11-15T00:49:21,511 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 327ed577e0f24b498c5f2d1f71e57e7a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731631757341 2024-11-15T00:49:21,511 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d902ff7b9e3a489bb2ade4e3d0fd35c1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731631757341 2024-11-15T00:49:21,511 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b82cdd2b11a43f0bf853065fbd2442d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731631757394 2024-11-15T00:49:21,511 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 849443a8e80741c89caa2912761d62e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731631757394 2024-11-15T00:49:21,512 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5b08b564d77b498f802147fc22e96e0d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731631758596 2024-11-15T00:49:21,512 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 442e141812024ed18cd467f78eb91e16, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731631758596 2024-11-15T00:49:21,512 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9691f31b837444d2a2e63bea194f78f6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631760826 2024-11-15T00:49:21,512 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3b0813c8f9094ecf9f166dbc48da24c9, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631760826 2024-11-15T00:49:21,524 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#B#compaction#172 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:21,525 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/1813d3c137b74accae448b29d4c6d440 is 50, key is test_row_0/B:col10/1731631760826/Put/seqid=0 2024-11-15T00:49:21,535 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#A#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:21,535 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/43078e7b21f9486e877fef2f5620faf8 is 50, key is test_row_0/A:col10/1731631760826/Put/seqid=0 2024-11-15T00:49:21,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742039_1215 (size=12139) 2024-11-15T00:49:21,548 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/1813d3c137b74accae448b29d4c6d440 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1813d3c137b74accae448b29d4c6d440 2024-11-15T00:49:21,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:21,551 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:21,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:49:21,555 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/B of 37711fc54bf4a3c70220e87c7201b430 into 1813d3c137b74accae448b29d4c6d440(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:21,555 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:21,555 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/B, priority=12, startTime=1731631761508; duration=0sec 2024-11-15T00:49:21,555 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:21,555 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:B 2024-11-15T00:49:21,555 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:21,557 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:21,557 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/C is initiating minor compaction (all files) 2024-11-15T00:49:21,557 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/C in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:21,557 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/324b54aadb86405c83ed710dd0525fbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/90a4902ae02b4e5bbd71480c706e8eaa, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4388d8edf4064515a9d3061c74ebc1c3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2afa2f4f5144fa2b6888535f6fa534c] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=46.9 K 2024-11-15T00:49:21,557 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 324b54aadb86405c83ed710dd0525fbf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1731631757341 2024-11-15T00:49:21,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742040_1216 (size=12139) 2024-11-15T00:49:21,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7656aa2ed33c427a81fb1cb637e04735 is 50, key is test_row_0/A:col10/1731631761550/Put/seqid=0 2024-11-15T00:49:21,559 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 90a4902ae02b4e5bbd71480c706e8eaa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731631757394 2024-11-15T00:49:21,561 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4388d8edf4064515a9d3061c74ebc1c3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=51, earliestPutTs=1731631758596 2024-11-15T00:49:21,562 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e2afa2f4f5144fa2b6888535f6fa534c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631760826 2024-11-15T00:49:21,566 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/43078e7b21f9486e877fef2f5620faf8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/43078e7b21f9486e877fef2f5620faf8 2024-11-15T00:49:21,568 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631821562, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,569 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,569 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631821563, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,570 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631821565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631821567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,571 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631821567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,571 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,572 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,572 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/A of 37711fc54bf4a3c70220e87c7201b430 into 43078e7b21f9486e877fef2f5620faf8(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:21,572 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:21,572 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/A, priority=12, startTime=1731631761508; duration=0sec 2024-11-15T00:49:21,572 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:21,572 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:A 2024-11-15T00:49:21,586 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742041_1217 (size=12001) 2024-11-15T00:49:21,592 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#C#compaction#175 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:21,593 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/78475894e74c4bad8c6273dd929bb62a is 50, key is test_row_0/C:col10/1731631760826/Put/seqid=0 2024-11-15T00:49:21,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:49:21,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T00:49:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:21,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:21,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:21,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:21,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742042_1218 (size=12139) 2024-11-15T00:49:21,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631821677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,681 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631821677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,681 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631821678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631821678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,682 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631821680, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,683 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,799 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T00:49:21,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:21,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:21,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:21,800 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:21,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:21,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:21,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:49:21,889 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631821886, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,890 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631821887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,890 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631821887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,891 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631821887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,891 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:21,891 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631821887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:21,892 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:21,956 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T00:49:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:21,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:21,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:21,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:21,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7656aa2ed33c427a81fb1cb637e04735 2024-11-15T00:49:21,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/aa0716b23afc4c10b3f4ce701caa176d is 50, key is test_row_0/B:col10/1731631761550/Put/seqid=0 2024-11-15T00:49:22,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742043_1219 (size=12001) 2024-11-15T00:49:22,054 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/78475894e74c4bad8c6273dd929bb62a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/78475894e74c4bad8c6273dd929bb62a 2024-11-15T00:49:22,060 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/C of 37711fc54bf4a3c70220e87c7201b430 into 78475894e74c4bad8c6273dd929bb62a(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:22,060 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:22,060 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/C, priority=12, startTime=1731631761508; duration=0sec 2024-11-15T00:49:22,060 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:22,060 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:C 2024-11-15T00:49:22,111 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T00:49:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:22,111 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:22,111 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:22,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:22,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:49:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631822197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,198 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631822197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631822197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,199 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631822198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631822198, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,200 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,264 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T00:49:22,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:22,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:22,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:22,265 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:22,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:22,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:22,405 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/aa0716b23afc4c10b3f4ce701caa176d 2024-11-15T00:49:22,418 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T00:49:22,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:22,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:22,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:22,419 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] handler.RSProcedureHandler(58): pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:22,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=69 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:22,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=69 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:22,432 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/d4c0016dc658432ba0a0d217d21670f2 is 50, key is test_row_0/C:col10/1731631761550/Put/seqid=0 2024-11-15T00:49:22,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742044_1220 (size=12001) 2024-11-15T00:49:22,452 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/d4c0016dc658432ba0a0d217d21670f2 2024-11-15T00:49:22,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7656aa2ed33c427a81fb1cb637e04735 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7656aa2ed33c427a81fb1cb637e04735 2024-11-15T00:49:22,469 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7656aa2ed33c427a81fb1cb637e04735, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T00:49:22,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/aa0716b23afc4c10b3f4ce701caa176d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/aa0716b23afc4c10b3f4ce701caa176d 2024-11-15T00:49:22,479 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/aa0716b23afc4c10b3f4ce701caa176d, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T00:49:22,480 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/d4c0016dc658432ba0a0d217d21670f2 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d4c0016dc658432ba0a0d217d21670f2 2024-11-15T00:49:22,490 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d4c0016dc658432ba0a0d217d21670f2, entries=150, sequenceid=89, filesize=11.7 K 2024-11-15T00:49:22,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 37711fc54bf4a3c70220e87c7201b430 in 940ms, sequenceid=89, compaction requested=false 2024-11-15T00:49:22,491 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:22,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=69 2024-11-15T00:49:22,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:22,573 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:49:22,590 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/72f7ba5904af4560947e635eab7f4dd5 is 50, key is test_row_0/A:col10/1731631761562/Put/seqid=0 2024-11-15T00:49:22,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742045_1221 (size=12001) 2024-11-15T00:49:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:49:22,710 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:22,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:22,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631822741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,750 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,751 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631822743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631822743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,757 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,758 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631822749, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631822750, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,764 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,774 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,857 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631822856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,858 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631822867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631822867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,871 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,871 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631822867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:22,879 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:22,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631822876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:22,880 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,027 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/72f7ba5904af4560947e635eab7f4dd5 2024-11-15T00:49:23,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/a2f79c91e78a4371a40c2b5ea2961aa7 is 50, key is test_row_0/B:col10/1731631761562/Put/seqid=0 2024-11-15T00:49:23,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742046_1222 (size=12001) 2024-11-15T00:49:23,068 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631823066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,068 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631823077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631823077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631823077, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,079 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,080 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,088 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631823086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,089 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631823377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631823387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,390 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631823387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,391 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631823388, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,400 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631823397, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,400 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,458 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/a2f79c91e78a4371a40c2b5ea2961aa7 2024-11-15T00:49:23,496 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/bf9470962a244ae6b476398a3b5a34e5 is 50, key is test_row_0/C:col10/1731631761562/Put/seqid=0 2024-11-15T00:49:23,502 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742047_1223 (size=12001) 2024-11-15T00:49:23,504 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=103 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/bf9470962a244ae6b476398a3b5a34e5 2024-11-15T00:49:23,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/72f7ba5904af4560947e635eab7f4dd5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/72f7ba5904af4560947e635eab7f4dd5 2024-11-15T00:49:23,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/72f7ba5904af4560947e635eab7f4dd5, entries=150, sequenceid=103, filesize=11.7 K 2024-11-15T00:49:23,521 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/a2f79c91e78a4371a40c2b5ea2961aa7 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a2f79c91e78a4371a40c2b5ea2961aa7 2024-11-15T00:49:23,528 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a2f79c91e78a4371a40c2b5ea2961aa7, entries=150, sequenceid=103, filesize=11.7 K 2024-11-15T00:49:23,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/bf9470962a244ae6b476398a3b5a34e5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/bf9470962a244ae6b476398a3b5a34e5 2024-11-15T00:49:23,534 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/bf9470962a244ae6b476398a3b5a34e5, entries=150, sequenceid=103, filesize=11.7 K 2024-11-15T00:49:23,536 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 37711fc54bf4a3c70220e87c7201b430 in 963ms, sequenceid=103, compaction requested=true 2024-11-15T00:49:23,536 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:23,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=69}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=69 2024-11-15T00:49:23,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=69 2024-11-15T00:49:23,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-15T00:49:23,552 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0490 sec 2024-11-15T00:49:23,559 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=68, table=TestAcidGuarantees in 2.0690 sec 2024-11-15T00:49:23,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=68 2024-11-15T00:49:23,637 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:23,638 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:23,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-15T00:49:23,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T00:49:23,641 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:23,642 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:23,642 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:23,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T00:49:23,798 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-15T00:49:23,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:23,798 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:49:23,808 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/23631f4bc6ae4899978ca3cf599b5591 is 50, key is test_row_0/A:col10/1731631762747/Put/seqid=0 2024-11-15T00:49:23,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742048_1224 (size=12001) 2024-11-15T00:49:23,858 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/23631f4bc6ae4899978ca3cf599b5591 2024-11-15T00:49:23,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/a8515176c57842b1a2c48bc65352f374 is 50, key is test_row_0/B:col10/1731631762747/Put/seqid=0 2024-11-15T00:49:23,891 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:23,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:23,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631823906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,910 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631823906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,911 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631823908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,913 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742049_1225 (size=12001) 2024-11-15T00:49:23,914 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/a8515176c57842b1a2c48bc65352f374 2024-11-15T00:49:23,915 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631823909, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:23,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631823910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,916 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:23,917 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,917 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:23,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T00:49:23,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/9236cb550a2648259afc939164113470 is 50, key is test_row_0/C:col10/1731631762747/Put/seqid=0 2024-11-15T00:49:24,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742050_1226 (size=12001) 2024-11-15T00:49:24,020 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631824018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,024 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631824027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631824027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,030 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,031 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631824226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,240 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631824237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,241 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,241 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631824237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T00:49:24,409 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=128 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/9236cb550a2648259afc939164113470 2024-11-15T00:49:24,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/23631f4bc6ae4899978ca3cf599b5591 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/23631f4bc6ae4899978ca3cf599b5591 2024-11-15T00:49:24,467 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/23631f4bc6ae4899978ca3cf599b5591, entries=150, sequenceid=128, filesize=11.7 K 2024-11-15T00:49:24,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/a8515176c57842b1a2c48bc65352f374 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a8515176c57842b1a2c48bc65352f374 2024-11-15T00:49:24,482 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a8515176c57842b1a2c48bc65352f374, entries=150, sequenceid=128, filesize=11.7 K 2024-11-15T00:49:24,483 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/9236cb550a2648259afc939164113470 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/9236cb550a2648259afc939164113470 2024-11-15T00:49:24,518 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/9236cb550a2648259afc939164113470, entries=150, sequenceid=128, filesize=11.7 K 2024-11-15T00:49:24,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-15T00:49:24,523 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 37711fc54bf4a3c70220e87c7201b430 in 724ms, sequenceid=128, compaction requested=true 2024-11-15T00:49:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:24,523 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-15T00:49:24,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-15T00:49:24,542 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-15T00:49:24,543 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 888 msec 2024-11-15T00:49:24,553 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:24,554 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T00:49:24,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 914 msec 2024-11-15T00:49:24,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:24,581 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/5985b1859e414920ad32d97e13c01b27 is 50, key is test_row_0/A:col10/1731631764551/Put/seqid=0 2024-11-15T00:49:24,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742051_1227 (size=14541) 2024-11-15T00:49:24,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631824664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631824669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,680 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,682 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631824671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,683 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,684 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-15T00:49:24,778 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:24,778 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:24,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-15T00:49:24,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:49:24,780 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:24,781 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:24,781 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:24,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631824787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,792 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631824787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,792 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631824787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,793 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,796 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:49:24,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631824917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,924 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 37711fc54bf4a3c70220e87c7201b430 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,925 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:24,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631824927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:24,935 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T00:49:24,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:24,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:24,935 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 37711fc54bf4a3c70220e87c7201b430 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2186 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:24,935 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:24,936 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:24,936 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:24,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:24,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,002 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,002 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631824996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631824997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,003 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631825007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,011 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,017 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/5985b1859e414920ad32d97e13c01b27 2024-11-15T00:49:25,042 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/2239d556e8624dbf936f384f56fe2e3b is 50, key is test_row_0/B:col10/1731631764551/Put/seqid=0 2024-11-15T00:49:25,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T00:49:25,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:25,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,090 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,090 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742052_1228 (size=12151) 2024-11-15T00:49:25,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/2239d556e8624dbf936f384f56fe2e3b 2024-11-15T00:49:25,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:49:25,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/207953f3a0f84922bf1f7bd4ab9bf789 is 50, key is test_row_0/C:col10/1731631764551/Put/seqid=0 2024-11-15T00:49:25,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742053_1229 (size=12151) 2024-11-15T00:49:25,244 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T00:49:25,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:25,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,244 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,244 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631825307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,310 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631825307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,310 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,320 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631825317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,397 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T00:49:25,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:25,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,398 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,398 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:49:25,551 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T00:49:25,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:25,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,552 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:25,580 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=142 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/207953f3a0f84922bf1f7bd4ab9bf789 2024-11-15T00:49:25,600 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/5985b1859e414920ad32d97e13c01b27 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/5985b1859e414920ad32d97e13c01b27 2024-11-15T00:49:25,608 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/5985b1859e414920ad32d97e13c01b27, entries=200, sequenceid=142, filesize=14.2 K 2024-11-15T00:49:25,610 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/2239d556e8624dbf936f384f56fe2e3b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/2239d556e8624dbf936f384f56fe2e3b 2024-11-15T00:49:25,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/2239d556e8624dbf936f384f56fe2e3b, entries=150, sequenceid=142, filesize=11.9 K 2024-11-15T00:49:25,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/207953f3a0f84922bf1f7bd4ab9bf789 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/207953f3a0f84922bf1f7bd4ab9bf789 2024-11-15T00:49:25,625 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/207953f3a0f84922bf1f7bd4ab9bf789, entries=150, sequenceid=142, filesize=11.9 K 2024-11-15T00:49:25,626 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 37711fc54bf4a3c70220e87c7201b430 in 1073ms, sequenceid=142, compaction requested=true 2024-11-15T00:49:25,626 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:25,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:25,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:25,626 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:49:25,626 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:49:25,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:25,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:25,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:25,626 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:25,627 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 62683 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:49:25,627 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/A is initiating minor compaction (all files) 2024-11-15T00:49:25,628 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/A in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,628 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/43078e7b21f9486e877fef2f5620faf8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7656aa2ed33c427a81fb1cb637e04735, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/72f7ba5904af4560947e635eab7f4dd5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/23631f4bc6ae4899978ca3cf599b5591, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/5985b1859e414920ad32d97e13c01b27] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=61.2 K 2024-11-15T00:49:25,629 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 43078e7b21f9486e877fef2f5620faf8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631760826 2024-11-15T00:49:25,629 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7656aa2ed33c427a81fb1cb637e04735, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731631760897 2024-11-15T00:49:25,630 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 72f7ba5904af4560947e635eab7f4dd5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731631761552 2024-11-15T00:49:25,630 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23631f4bc6ae4899978ca3cf599b5591, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731631762737 2024-11-15T00:49:25,630 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5985b1859e414920ad32d97e13c01b27, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731631763909 2024-11-15T00:49:25,634 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60293 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:49:25,634 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/B is initiating minor compaction (all files) 2024-11-15T00:49:25,634 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/B in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,634 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1813d3c137b74accae448b29d4c6d440, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/aa0716b23afc4c10b3f4ce701caa176d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a2f79c91e78a4371a40c2b5ea2961aa7, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a8515176c57842b1a2c48bc65352f374, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/2239d556e8624dbf936f384f56fe2e3b] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=58.9 K 2024-11-15T00:49:25,635 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1813d3c137b74accae448b29d4c6d440, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631760826 2024-11-15T00:49:25,635 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting aa0716b23afc4c10b3f4ce701caa176d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731631760897 2024-11-15T00:49:25,635 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a2f79c91e78a4371a40c2b5ea2961aa7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731631761552 2024-11-15T00:49:25,636 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a8515176c57842b1a2c48bc65352f374, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731631762737 2024-11-15T00:49:25,636 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2239d556e8624dbf936f384f56fe2e3b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731631764544 2024-11-15T00:49:25,651 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#A#compaction#187 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:25,652 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/30b69031f53e4ce1b0dc4d1232991d4d is 50, key is test_row_0/A:col10/1731631764551/Put/seqid=0 2024-11-15T00:49:25,664 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#B#compaction#188 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:25,664 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/7f2187ae8a9640168ff4c16f887aaf70 is 50, key is test_row_0/B:col10/1731631764551/Put/seqid=0 2024-11-15T00:49:25,677 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742054_1230 (size=12459) 2024-11-15T00:49:25,685 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/30b69031f53e4ce1b0dc4d1232991d4d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/30b69031f53e4ce1b0dc4d1232991d4d 2024-11-15T00:49:25,690 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/A of 37711fc54bf4a3c70220e87c7201b430 into 30b69031f53e4ce1b0dc4d1232991d4d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:25,690 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:25,690 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/A, priority=11, startTime=1731631765626; duration=0sec 2024-11-15T00:49:25,690 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:25,690 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:A 2024-11-15T00:49:25,690 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:49:25,692 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60293 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:49:25,692 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/C is initiating minor compaction (all files) 2024-11-15T00:49:25,693 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/C in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,693 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/78475894e74c4bad8c6273dd929bb62a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d4c0016dc658432ba0a0d217d21670f2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/bf9470962a244ae6b476398a3b5a34e5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/9236cb550a2648259afc939164113470, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/207953f3a0f84922bf1f7bd4ab9bf789] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=58.9 K 2024-11-15T00:49:25,693 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 78475894e74c4bad8c6273dd929bb62a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=63, earliestPutTs=1731631760826 2024-11-15T00:49:25,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742055_1231 (size=12459) 2024-11-15T00:49:25,694 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4c0016dc658432ba0a0d217d21670f2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1731631760897 2024-11-15T00:49:25,694 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bf9470962a244ae6b476398a3b5a34e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=103, earliestPutTs=1731631761552 2024-11-15T00:49:25,694 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9236cb550a2648259afc939164113470, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=128, earliestPutTs=1731631762737 2024-11-15T00:49:25,695 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 207953f3a0f84922bf1f7bd4ab9bf789, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731631764544 2024-11-15T00:49:25,703 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/7f2187ae8a9640168ff4c16f887aaf70 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/7f2187ae8a9640168ff4c16f887aaf70 2024-11-15T00:49:25,710 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#C#compaction#189 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:25,711 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/B of 37711fc54bf4a3c70220e87c7201b430 into 7f2187ae8a9640168ff4c16f887aaf70(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:25,711 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:25,711 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/B, priority=11, startTime=1731631765626; duration=0sec 2024-11-15T00:49:25,711 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:25,711 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/d0d1152c40c6447a93507897b83e7ca7 is 50, key is test_row_0/C:col10/1731631764551/Put/seqid=0 2024-11-15T00:49:25,711 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:B 2024-11-15T00:49:25,711 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-15T00:49:25,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:25,712 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T00:49:25,718 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/593d94b915364829b67c87946a0eacbf is 50, key is test_row_0/A:col10/1731631764661/Put/seqid=0 2024-11-15T00:49:25,719 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742056_1232 (size=12459) 2024-11-15T00:49:25,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742057_1233 (size=12151) 2024-11-15T00:49:25,748 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/593d94b915364829b67c87946a0eacbf 2024-11-15T00:49:25,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/1d76b6d11dcf49eabf2777ab1dcac95b is 50, key is test_row_0/B:col10/1731631764661/Put/seqid=0 2024-11-15T00:49:25,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742058_1234 (size=12151) 2024-11-15T00:49:25,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:25,820 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:25,854 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631825850, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,855 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631825851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631825851, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,862 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,863 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,864 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:49:25,960 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631825956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,960 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,961 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,969 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631825967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,970 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:25,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:25,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631825967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:25,971 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:26,140 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/d0d1152c40c6447a93507897b83e7ca7 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d0d1152c40c6447a93507897b83e7ca7 2024-11-15T00:49:26,159 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/C of 37711fc54bf4a3c70220e87c7201b430 into d0d1152c40c6447a93507897b83e7ca7(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:26,159 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:26,159 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/C, priority=11, startTime=1731631765626; duration=0sec 2024-11-15T00:49:26,160 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:26,160 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:C 2024-11-15T00:49:26,168 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:26,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631826166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,169 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:26,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:26,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631826177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:26,179 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:26,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631826177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:26,221 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/1d76b6d11dcf49eabf2777ab1dcac95b 2024-11-15T00:49:26,239 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/b446d26ac5d7467ba82e1af1a619bcbf is 50, key is test_row_0/C:col10/1731631764661/Put/seqid=0 2024-11-15T00:49:26,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742059_1235 (size=12151) 2024-11-15T00:49:26,477 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:26,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631826477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:26,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,478 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:26,490 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631826487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:26,490 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:26,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631826487, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:26,657 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/b446d26ac5d7467ba82e1af1a619bcbf 2024-11-15T00:49:26,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/593d94b915364829b67c87946a0eacbf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/593d94b915364829b67c87946a0eacbf 2024-11-15T00:49:26,674 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/593d94b915364829b67c87946a0eacbf, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T00:49:26,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/1d76b6d11dcf49eabf2777ab1dcac95b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1d76b6d11dcf49eabf2777ab1dcac95b 2024-11-15T00:49:26,680 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1d76b6d11dcf49eabf2777ab1dcac95b, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T00:49:26,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/b446d26ac5d7467ba82e1af1a619bcbf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b446d26ac5d7467ba82e1af1a619bcbf 2024-11-15T00:49:26,691 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b446d26ac5d7467ba82e1af1a619bcbf, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T00:49:26,692 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 37711fc54bf4a3c70220e87c7201b430 in 980ms, sequenceid=166, compaction requested=false 2024-11-15T00:49:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:26,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-15T00:49:26,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-15T00:49:26,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-15T00:49:26,697 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9130 sec 2024-11-15T00:49:26,701 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 1.9210 sec 2024-11-15T00:49:26,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-15T00:49:26,937 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:26,937 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-15T00:49:26,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:49:26,939 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:26,939 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:26,939 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:26,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:26,949 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:26,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T00:49:26,959 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7465e83ba09a4c21aef96eb4ee0bb345 is 50, key is test_row_0/A:col10/1731631765840/Put/seqid=0 2024-11-15T00:49:26,986 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742060_1236 (size=14541) 2024-11-15T00:49:26,987 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7465e83ba09a4c21aef96eb4ee0bb345 2024-11-15T00:49:26,995 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:26,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631826992, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:26,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,995 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:26,997 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/39efdde218504eccbe29fb60323e26a5 is 50, key is test_row_0/B:col10/1731631765840/Put/seqid=0 2024-11-15T00:49:26,997 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:26,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631826993, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:26,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:26,998 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:26,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:26,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631826995, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631826997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,000 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631826997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,000 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742061_1237 (size=12151) 2024-11-15T00:49:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:49:27,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:49:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:27,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631827107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631827107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,109 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,109 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631827107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:49:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:49:27,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631827317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,321 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631827317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631827317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:49:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/39efdde218504eccbe29fb60323e26a5 2024-11-15T00:49:27,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/2009be1dc7204c28a6b7d28d8c1f7298 is 50, key is test_row_0/C:col10/1731631765840/Put/seqid=0 2024-11-15T00:49:27,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742062_1238 (size=12151) 2024-11-15T00:49:27,554 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:49:27,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:27,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,555 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:49:27,629 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631827626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,630 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631827626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,632 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631827627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:27,633 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:27,708 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:49:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:27,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,708 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:27,843 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=183 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/2009be1dc7204c28a6b7d28d8c1f7298 2024-11-15T00:49:27,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7465e83ba09a4c21aef96eb4ee0bb345 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7465e83ba09a4c21aef96eb4ee0bb345 2024-11-15T00:49:27,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7465e83ba09a4c21aef96eb4ee0bb345, entries=200, sequenceid=183, filesize=14.2 K 2024-11-15T00:49:27,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/39efdde218504eccbe29fb60323e26a5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/39efdde218504eccbe29fb60323e26a5 2024-11-15T00:49:27,855 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/39efdde218504eccbe29fb60323e26a5, entries=150, sequenceid=183, filesize=11.9 K 2024-11-15T00:49:27,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/2009be1dc7204c28a6b7d28d8c1f7298 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2009be1dc7204c28a6b7d28d8c1f7298 2024-11-15T00:49:27,860 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2009be1dc7204c28a6b7d28d8c1f7298, entries=150, sequenceid=183, filesize=11.9 K 2024-11-15T00:49:27,861 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 37711fc54bf4a3c70220e87c7201b430 in 912ms, sequenceid=183, compaction requested=true 2024-11-15T00:49:27,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-15T00:49:27,861 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:27,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:27,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:27,861 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:27,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:27,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:27,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:27,861 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:27,861 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:27,861 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-15T00:49:27,862 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39151 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:27,862 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/A is initiating minor compaction (all files) 2024-11-15T00:49:27,862 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/A in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,862 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/30b69031f53e4ce1b0dc4d1232991d4d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/593d94b915364829b67c87946a0eacbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7465e83ba09a4c21aef96eb4ee0bb345] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=38.2 K 2024-11-15T00:49:27,863 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:27,863 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/B is initiating minor compaction (all files) 2024-11-15T00:49:27,863 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/B in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,863 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/7f2187ae8a9640168ff4c16f887aaf70, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1d76b6d11dcf49eabf2777ab1dcac95b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/39efdde218504eccbe29fb60323e26a5] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=35.9 K 2024-11-15T00:49:27,863 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 30b69031f53e4ce1b0dc4d1232991d4d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731631764544 2024-11-15T00:49:27,863 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f2187ae8a9640168ff4c16f887aaf70, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731631764544 2024-11-15T00:49:27,864 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 593d94b915364829b67c87946a0eacbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631764659 2024-11-15T00:49:27,865 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d76b6d11dcf49eabf2777ab1dcac95b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631764659 2024-11-15T00:49:27,865 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7465e83ba09a4c21aef96eb4ee0bb345, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731631765840 2024-11-15T00:49:27,865 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 39efdde218504eccbe29fb60323e26a5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731631765840 2024-11-15T00:49:27,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/283f38e6d81a42e39d83a6c95d195cf4 is 50, key is test_row_0/A:col10/1731631766987/Put/seqid=0 2024-11-15T00:49:27,873 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#B#compaction#197 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:27,873 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#A#compaction#198 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:27,874 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/35b8d36006af438da67751cd34a205b8 is 50, key is test_row_0/A:col10/1731631765840/Put/seqid=0 2024-11-15T00:49:27,874 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/e8316cc6ec694e4fa6ff4fbdec76a9eb is 50, key is test_row_0/B:col10/1731631765840/Put/seqid=0 2024-11-15T00:49:27,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742065_1241 (size=12561) 2024-11-15T00:49:27,907 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/e8316cc6ec694e4fa6ff4fbdec76a9eb as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e8316cc6ec694e4fa6ff4fbdec76a9eb 2024-11-15T00:49:27,913 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/B of 37711fc54bf4a3c70220e87c7201b430 into e8316cc6ec694e4fa6ff4fbdec76a9eb(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:27,913 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:27,913 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/B, priority=13, startTime=1731631767861; duration=0sec 2024-11-15T00:49:27,913 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:27,913 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:B 2024-11-15T00:49:27,913 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:27,914 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36761 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:27,914 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/C is initiating minor compaction (all files) 2024-11-15T00:49:27,914 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/C in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:27,914 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d0d1152c40c6447a93507897b83e7ca7, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b446d26ac5d7467ba82e1af1a619bcbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2009be1dc7204c28a6b7d28d8c1f7298] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=35.9 K 2024-11-15T00:49:27,915 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d0d1152c40c6447a93507897b83e7ca7, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=142, earliestPutTs=1731631764544 2024-11-15T00:49:27,916 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b446d26ac5d7467ba82e1af1a619bcbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631764659 2024-11-15T00:49:27,916 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2009be1dc7204c28a6b7d28d8c1f7298, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731631765840 2024-11-15T00:49:27,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742063_1239 (size=12151) 2024-11-15T00:49:27,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742064_1240 (size=12561) 2024-11-15T00:49:27,924 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#C#compaction#199 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:27,925 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/f511fcb352844bd68e33bea86d7d58ee is 50, key is test_row_0/C:col10/1731631765840/Put/seqid=0 2024-11-15T00:49:27,928 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742066_1242 (size=12561) 2024-11-15T00:49:28,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:28,009 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:28,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631828040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,045 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631828041, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,045 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,046 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:49:28,139 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631828136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631828136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,140 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,149 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631828147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631828157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,160 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631828157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,160 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,318 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/283f38e6d81a42e39d83a6c95d195cf4 2024-11-15T00:49:28,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/0c04fd1ee00249cc84136e828ecd67c3 is 50, key is test_row_0/B:col10/1731631766987/Put/seqid=0 2024-11-15T00:49:28,341 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/f511fcb352844bd68e33bea86d7d58ee as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/f511fcb352844bd68e33bea86d7d58ee 2024-11-15T00:49:28,341 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/35b8d36006af438da67751cd34a205b8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/35b8d36006af438da67751cd34a205b8 2024-11-15T00:49:28,347 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/C of 37711fc54bf4a3c70220e87c7201b430 into f511fcb352844bd68e33bea86d7d58ee(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:28,347 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:28,347 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/C, priority=13, startTime=1731631767861; duration=0sec 2024-11-15T00:49:28,347 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:28,347 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:C 2024-11-15T00:49:28,348 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/A of 37711fc54bf4a3c70220e87c7201b430 into 35b8d36006af438da67751cd34a205b8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:28,348 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:28,348 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/A, priority=13, startTime=1731631767861; duration=0sec 2024-11-15T00:49:28,348 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:28,348 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:A 2024-11-15T00:49:28,369 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631828367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631828367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,370 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,374 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742067_1243 (size=12151) 2024-11-15T00:49:28,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631828676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631828677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,680 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:28,774 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/0c04fd1ee00249cc84136e828ecd67c3 2024-11-15T00:49:28,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/522951a0e4bb47ce9079a185966cfcbf is 50, key is test_row_0/C:col10/1731631766987/Put/seqid=0 2024-11-15T00:49:28,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742068_1244 (size=12151) 2024-11-15T00:49:29,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:49:29,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:29,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631829147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:29,148 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 37711fc54bf4a3c70220e87c7201b430 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:29,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:29,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:29,149 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:29,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:29,158 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631829156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:29,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631829156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:29,159 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 37711fc54bf4a3c70220e87c7201b430 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2166 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:29,159 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:29,160 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:29,164 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 37711fc54bf4a3c70220e87c7201b430 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:29,165 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:29,187 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=203 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/522951a0e4bb47ce9079a185966cfcbf 2024-11-15T00:49:29,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:29,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631829187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:29,189 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:29,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631829187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:29,189 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:29,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/283f38e6d81a42e39d83a6c95d195cf4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/283f38e6d81a42e39d83a6c95d195cf4 2024-11-15T00:49:29,196 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/283f38e6d81a42e39d83a6c95d195cf4, entries=150, sequenceid=203, filesize=11.9 K 2024-11-15T00:49:29,198 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/0c04fd1ee00249cc84136e828ecd67c3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0c04fd1ee00249cc84136e828ecd67c3 2024-11-15T00:49:29,201 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0c04fd1ee00249cc84136e828ecd67c3, entries=150, sequenceid=203, filesize=11.9 K 2024-11-15T00:49:29,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/522951a0e4bb47ce9079a185966cfcbf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/522951a0e4bb47ce9079a185966cfcbf 2024-11-15T00:49:29,207 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/522951a0e4bb47ce9079a185966cfcbf, entries=150, sequenceid=203, filesize=11.9 K 2024-11-15T00:49:29,207 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 37711fc54bf4a3c70220e87c7201b430 in 1346ms, sequenceid=203, compaction requested=false 2024-11-15T00:49:29,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:29,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:29,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-15T00:49:29,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-15T00:49:29,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-15T00:49:29,210 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2700 sec 2024-11-15T00:49:29,213 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 2.2740 sec 2024-11-15T00:49:30,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:30,199 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:30,199 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-15T00:49:30,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/9048f928942849c5b93af3e08b780827 is 50, key is test_row_0/A:col10/1731631770197/Put/seqid=0 2024-11-15T00:49:30,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742069_1245 (size=12151) 2024-11-15T00:49:30,284 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/9048f928942849c5b93af3e08b780827 2024-11-15T00:49:30,309 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:30,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631830303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,311 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:30,311 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:30,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631830307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:30,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,313 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:30,325 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/414e9cf22bca4a21ab54d6ece1271cbb is 50, key is test_row_0/B:col10/1731631770197/Put/seqid=0 2024-11-15T00:49:30,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742070_1246 (size=12151) 2024-11-15T00:49:30,423 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:30,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631830420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:30,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631830420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:30,424 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:30,629 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631830627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,630 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:30,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631830627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:30,630 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:30,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/414e9cf22bca4a21ab54d6ece1271cbb 2024-11-15T00:49:30,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/86b1a288185b408b973b67ac6fa976b7 is 50, key is test_row_0/C:col10/1731631770197/Put/seqid=0 2024-11-15T00:49:30,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742071_1247 (size=12151) 2024-11-15T00:49:30,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=223 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/86b1a288185b408b973b67ac6fa976b7 2024-11-15T00:49:30,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/9048f928942849c5b93af3e08b780827 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9048f928942849c5b93af3e08b780827 2024-11-15T00:49:30,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9048f928942849c5b93af3e08b780827, entries=150, sequenceid=223, filesize=11.9 K 2024-11-15T00:49:30,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/414e9cf22bca4a21ab54d6ece1271cbb as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/414e9cf22bca4a21ab54d6ece1271cbb 2024-11-15T00:49:30,849 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/414e9cf22bca4a21ab54d6ece1271cbb, entries=150, sequenceid=223, filesize=11.9 K 2024-11-15T00:49:30,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/86b1a288185b408b973b67ac6fa976b7 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/86b1a288185b408b973b67ac6fa976b7 2024-11-15T00:49:30,859 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/86b1a288185b408b973b67ac6fa976b7, entries=150, sequenceid=223, filesize=11.9 K 2024-11-15T00:49:30,868 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 37711fc54bf4a3c70220e87c7201b430 in 669ms, sequenceid=223, compaction requested=true 2024-11-15T00:49:30,868 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:30,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:30,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:30,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:30,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:49:30,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:30,869 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T00:49:30,869 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:30,869 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:30,893 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:30,893 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/C is initiating minor compaction (all files) 2024-11-15T00:49:30,893 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/C in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:30,893 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/f511fcb352844bd68e33bea86d7d58ee, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/522951a0e4bb47ce9079a185966cfcbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/86b1a288185b408b973b67ac6fa976b7] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=36.0 K 2024-11-15T00:49:30,893 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:30,893 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/A is initiating minor compaction (all files) 2024-11-15T00:49:30,893 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/A in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:30,893 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/35b8d36006af438da67751cd34a205b8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/283f38e6d81a42e39d83a6c95d195cf4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9048f928942849c5b93af3e08b780827] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=36.0 K 2024-11-15T00:49:30,894 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 35b8d36006af438da67751cd34a205b8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731631765840 2024-11-15T00:49:30,894 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f511fcb352844bd68e33bea86d7d58ee, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731631765840 2024-11-15T00:49:30,899 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 522951a0e4bb47ce9079a185966cfcbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731631766987 2024-11-15T00:49:30,899 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 283f38e6d81a42e39d83a6c95d195cf4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731631766987 2024-11-15T00:49:30,904 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 86b1a288185b408b973b67ac6fa976b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731631768020 2024-11-15T00:49:30,905 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9048f928942849c5b93af3e08b780827, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731631768020 2024-11-15T00:49:30,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:30,956 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:30,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T00:49:30,958 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#A#compaction#205 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:30,959 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7a764cd0cbaa4e00868045886ba90b90 is 50, key is test_row_0/A:col10/1731631770197/Put/seqid=0 2024-11-15T00:49:30,970 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#C#compaction#206 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:30,970 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/2da096d7c3414da5b9212b67478c3a77 is 50, key is test_row_0/C:col10/1731631770197/Put/seqid=0 2024-11-15T00:49:31,001 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/eeefd778716f4cf3b824b4d8c92e9d7e is 50, key is test_row_0/A:col10/1731631770946/Put/seqid=0 2024-11-15T00:49:31,005 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742073_1249 (size=12663) 2024-11-15T00:49:31,011 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742072_1248 (size=12663) 2024-11-15T00:49:31,013 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/2da096d7c3414da5b9212b67478c3a77 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2da096d7c3414da5b9212b67478c3a77 2024-11-15T00:49:31,019 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7a764cd0cbaa4e00868045886ba90b90 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7a764cd0cbaa4e00868045886ba90b90 2024-11-15T00:49:31,025 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631831018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,025 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/C of 37711fc54bf4a3c70220e87c7201b430 into 2da096d7c3414da5b9212b67478c3a77(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:31,025 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,025 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/C, priority=13, startTime=1731631770869; duration=0sec 2024-11-15T00:49:31,025 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,025 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:31,025 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:C 2024-11-15T00:49:31,026 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:31,026 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631831019, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,027 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,031 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:31,031 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/B is initiating minor compaction (all files) 2024-11-15T00:49:31,031 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/A of 37711fc54bf4a3c70220e87c7201b430 into 7a764cd0cbaa4e00868045886ba90b90(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:31,031 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/B in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,031 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:31,031 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/A, priority=13, startTime=1731631770868; duration=0sec 2024-11-15T00:49:31,031 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e8316cc6ec694e4fa6ff4fbdec76a9eb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0c04fd1ee00249cc84136e828ecd67c3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/414e9cf22bca4a21ab54d6ece1271cbb] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=36.0 K 2024-11-15T00:49:31,031 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:31,031 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:A 2024-11-15T00:49:31,032 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e8316cc6ec694e4fa6ff4fbdec76a9eb, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=183, earliestPutTs=1731631765840 2024-11-15T00:49:31,032 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c04fd1ee00249cc84136e828ecd67c3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=203, earliestPutTs=1731631766987 2024-11-15T00:49:31,033 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 414e9cf22bca4a21ab54d6ece1271cbb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731631768020 2024-11-15T00:49:31,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742074_1250 (size=12151) 2024-11-15T00:49:31,049 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/eeefd778716f4cf3b824b4d8c92e9d7e 2024-11-15T00:49:31,060 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#B#compaction#208 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:31,060 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/f8c35c13f95d4a118a75eb7d9aaad9b1 is 50, key is test_row_0/B:col10/1731631770197/Put/seqid=0 2024-11-15T00:49:31,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/639dc34b770d41048a9a3a001d5a428a is 50, key is test_row_0/B:col10/1731631770946/Put/seqid=0 2024-11-15T00:49:31,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-15T00:49:31,101 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:31,101 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:31,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-15T00:49:31,105 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:31,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:49:31,106 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:31,106 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:31,134 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742075_1251 (size=12663) 2024-11-15T00:49:31,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742076_1252 (size=12151) 2024-11-15T00:49:31,141 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631831137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,141 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,142 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631831137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,142 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631831157, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,160 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 37711fc54bf4a3c70220e87c7201b430 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4168 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,161 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631831178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,179 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 37711fc54bf4a3c70220e87c7201b430 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,180 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631831180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,184 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 37711fc54bf4a3c70220e87c7201b430 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4189 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,184 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:49:31,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T00:49:31,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:31,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,269 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,269 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,349 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631831347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631831347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,350 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,422 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T00:49:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,422 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,422 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:49:31,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/639dc34b770d41048a9a3a001d5a428a 2024-11-15T00:49:31,542 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/f8c35c13f95d4a118a75eb7d9aaad9b1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/f8c35c13f95d4a118a75eb7d9aaad9b1 2024-11-15T00:49:31,560 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/B of 37711fc54bf4a3c70220e87c7201b430 into f8c35c13f95d4a118a75eb7d9aaad9b1(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:31,560 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:31,560 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/B, priority=13, startTime=1731631770869; duration=0sec 2024-11-15T00:49:31,561 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:31,561 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:B 2024-11-15T00:49:31,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/2b8a5b58f1014ccca0a6c2a2a59e02d3 is 50, key is test_row_0/C:col10/1731631770946/Put/seqid=0 2024-11-15T00:49:31,575 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T00:49:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742077_1253 (size=12151) 2024-11-15T00:49:31,662 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631831658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:31,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631831658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:31,663 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:31,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T00:49:31,731 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:49:31,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T00:49:31,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:31,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:31,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] handler.RSProcedureHandler(58): pid=77 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=77 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:31,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=77 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:32,008 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/2b8a5b58f1014ccca0a6c2a2a59e02d3 2024-11-15T00:49:32,013 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/eeefd778716f4cf3b824b4d8c92e9d7e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/eeefd778716f4cf3b824b4d8c92e9d7e 2024-11-15T00:49:32,019 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/eeefd778716f4cf3b824b4d8c92e9d7e, entries=150, sequenceid=244, filesize=11.9 K 2024-11-15T00:49:32,020 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/639dc34b770d41048a9a3a001d5a428a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/639dc34b770d41048a9a3a001d5a428a 2024-11-15T00:49:32,025 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/639dc34b770d41048a9a3a001d5a428a, entries=150, sequenceid=244, filesize=11.9 K 2024-11-15T00:49:32,026 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/2b8a5b58f1014ccca0a6c2a2a59e02d3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2b8a5b58f1014ccca0a6c2a2a59e02d3 2024-11-15T00:49:32,031 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2b8a5b58f1014ccca0a6c2a2a59e02d3, entries=150, sequenceid=244, filesize=11.9 K 2024-11-15T00:49:32,032 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=87.22 KB/89310 for 37711fc54bf4a3c70220e87c7201b430 in 1076ms, sequenceid=244, compaction requested=false 2024-11-15T00:49:32,032 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:32,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-15T00:49:32,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:32,039 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T00:49:32,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/b1581f6f278242c6bb3d838c961d444b is 50, key is test_row_0/A:col10/1731631771017/Put/seqid=0 2024-11-15T00:49:32,073 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742078_1254 (size=12251) 2024-11-15T00:49:32,075 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/b1581f6f278242c6bb3d838c961d444b 2024-11-15T00:49:32,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/13cbbc6617614c948bdbb10e0427157b is 50, key is test_row_0/B:col10/1731631771017/Put/seqid=0 2024-11-15T00:49:32,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742079_1255 (size=12251) 2024-11-15T00:49:32,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:32,169 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:32,214 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:32,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631832209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:32,215 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:32,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631832211, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,216 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:49:32,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631832327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:32,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631832327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:32,329 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:32,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,330 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:32,513 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/13cbbc6617614c948bdbb10e0427157b 2024-11-15T00:49:32,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/ae9962223f1a4e23aa8195cac9ea77c9 is 50, key is test_row_0/C:col10/1731631771017/Put/seqid=0 2024-11-15T00:49:32,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631832537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:32,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:32,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,541 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:32,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631832537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:32,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742080_1256 (size=12251) 2024-11-15T00:49:32,850 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:32,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631832847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:32,851 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:32,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631832847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:32,851 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:32,947 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=263 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/ae9962223f1a4e23aa8195cac9ea77c9 2024-11-15T00:49:32,952 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/b1581f6f278242c6bb3d838c961d444b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b1581f6f278242c6bb3d838c961d444b 2024-11-15T00:49:32,957 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b1581f6f278242c6bb3d838c961d444b, entries=150, sequenceid=263, filesize=12.0 K 2024-11-15T00:49:32,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/13cbbc6617614c948bdbb10e0427157b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/13cbbc6617614c948bdbb10e0427157b 2024-11-15T00:49:32,965 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/13cbbc6617614c948bdbb10e0427157b, entries=150, sequenceid=263, filesize=12.0 K 2024-11-15T00:49:32,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/ae9962223f1a4e23aa8195cac9ea77c9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/ae9962223f1a4e23aa8195cac9ea77c9 2024-11-15T00:49:32,980 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/ae9962223f1a4e23aa8195cac9ea77c9, entries=150, sequenceid=263, filesize=12.0 K 2024-11-15T00:49:32,981 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 37711fc54bf4a3c70220e87c7201b430 in 942ms, sequenceid=263, compaction requested=true 2024-11-15T00:49:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:32,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-15T00:49:32,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-15T00:49:32,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-15T00:49:32,986 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8770 sec 2024-11-15T00:49:32,989 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.8860 sec 2024-11-15T00:49:33,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-15T00:49:33,257 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:33,257 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-15T00:49:33,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T00:49:33,259 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:33,259 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:33,259 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:33,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:33,359 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:33,359 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T00:49:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T00:49:33,390 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/f88c3d1575b94782b261d30ba88be4ef is 50, key is test_row_0/A:col10/1731631772188/Put/seqid=0 2024-11-15T00:49:33,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742081_1257 (size=14741) 2024-11-15T00:49:33,402 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/f88c3d1575b94782b261d30ba88be4ef 2024-11-15T00:49:33,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:33,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631833403, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:33,411 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T00:49:33,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:33,411 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:33,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:33,412 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:33,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:33,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:33,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631833408, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,414 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:33,438 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/d61c2b7853b447429bcf688676375244 is 50, key is test_row_0/B:col10/1731631772188/Put/seqid=0 2024-11-15T00:49:33,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742082_1258 (size=12301) 2024-11-15T00:49:33,481 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/d61c2b7853b447429bcf688676375244 2024-11-15T00:49:33,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/c4e3e3ea69cb47668a525cca0ac2b7a7 is 50, key is test_row_0/C:col10/1731631772188/Put/seqid=0 2024-11-15T00:49:33,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:33,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631833519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,527 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:33,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:33,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631833524, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,531 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,532 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:33,557 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742083_1259 (size=12301) 2024-11-15T00:49:33,558 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/c4e3e3ea69cb47668a525cca0ac2b7a7 2024-11-15T00:49:33,564 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/f88c3d1575b94782b261d30ba88be4ef as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/f88c3d1575b94782b261d30ba88be4ef 2024-11-15T00:49:33,565 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T00:49:33,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:33,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:33,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:33,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:33,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:33,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:33,570 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/f88c3d1575b94782b261d30ba88be4ef, entries=200, sequenceid=284, filesize=14.4 K 2024-11-15T00:49:33,573 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/d61c2b7853b447429bcf688676375244 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d61c2b7853b447429bcf688676375244 2024-11-15T00:49:33,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T00:49:33,577 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d61c2b7853b447429bcf688676375244, entries=150, sequenceid=284, filesize=12.0 K 2024-11-15T00:49:33,578 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/c4e3e3ea69cb47668a525cca0ac2b7a7 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/c4e3e3ea69cb47668a525cca0ac2b7a7 2024-11-15T00:49:33,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/c4e3e3ea69cb47668a525cca0ac2b7a7, entries=150, sequenceid=284, filesize=12.0 K 2024-11-15T00:49:33,583 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 37711fc54bf4a3c70220e87c7201b430 in 224ms, sequenceid=284, compaction requested=true 2024-11-15T00:49:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:33,583 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:33,583 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:33,583 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:33,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:33,584 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:33,585 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 51806 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:33,585 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:33,585 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/B is initiating minor compaction (all files) 2024-11-15T00:49:33,585 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/A is initiating minor compaction (all files) 2024-11-15T00:49:33,585 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/A in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:33,585 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/B in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:33,585 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7a764cd0cbaa4e00868045886ba90b90, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/eeefd778716f4cf3b824b4d8c92e9d7e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b1581f6f278242c6bb3d838c961d444b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/f88c3d1575b94782b261d30ba88be4ef] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=50.6 K 2024-11-15T00:49:33,585 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/f8c35c13f95d4a118a75eb7d9aaad9b1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/639dc34b770d41048a9a3a001d5a428a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/13cbbc6617614c948bdbb10e0427157b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d61c2b7853b447429bcf688676375244] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=48.2 K 2024-11-15T00:49:33,585 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a764cd0cbaa4e00868045886ba90b90, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731631768020 2024-11-15T00:49:33,585 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f8c35c13f95d4a118a75eb7d9aaad9b1, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731631768020 2024-11-15T00:49:33,585 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting eeefd778716f4cf3b824b4d8c92e9d7e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731631770267 2024-11-15T00:49:33,586 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 639dc34b770d41048a9a3a001d5a428a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731631770267 2024-11-15T00:49:33,586 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1581f6f278242c6bb3d838c961d444b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731631770998 2024-11-15T00:49:33,586 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f88c3d1575b94782b261d30ba88be4ef, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731631772188 2024-11-15T00:49:33,586 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 13cbbc6617614c948bdbb10e0427157b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731631770998 2024-11-15T00:49:33,586 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d61c2b7853b447429bcf688676375244, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731631772188 2024-11-15T00:49:33,603 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#B#compaction#217 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:33,604 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/ae3afd73f5ff40ec8fc1d89d553a1bbb is 50, key is test_row_0/B:col10/1731631772188/Put/seqid=0 2024-11-15T00:49:33,608 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#A#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:33,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742084_1260 (size=12949) 2024-11-15T00:49:33,608 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/95869ff8b6174c7c8a822a87b4fef49a is 50, key is test_row_0/A:col10/1731631772188/Put/seqid=0 2024-11-15T00:49:33,618 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/ae3afd73f5ff40ec8fc1d89d553a1bbb as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ae3afd73f5ff40ec8fc1d89d553a1bbb 2024-11-15T00:49:33,624 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/B of 37711fc54bf4a3c70220e87c7201b430 into ae3afd73f5ff40ec8fc1d89d553a1bbb(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:33,624 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:33,624 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/B, priority=12, startTime=1731631773583; duration=0sec 2024-11-15T00:49:33,624 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:33,624 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:B 2024-11-15T00:49:33,624 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:33,627 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49366 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:33,627 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/C is initiating minor compaction (all files) 2024-11-15T00:49:33,628 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/C in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:33,628 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2da096d7c3414da5b9212b67478c3a77, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2b8a5b58f1014ccca0a6c2a2a59e02d3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/ae9962223f1a4e23aa8195cac9ea77c9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/c4e3e3ea69cb47668a525cca0ac2b7a7] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=48.2 K 2024-11-15T00:49:33,629 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da096d7c3414da5b9212b67478c3a77, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=223, earliestPutTs=1731631768020 2024-11-15T00:49:33,630 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b8a5b58f1014ccca0a6c2a2a59e02d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731631770267 2024-11-15T00:49:33,630 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ae9962223f1a4e23aa8195cac9ea77c9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=263, earliestPutTs=1731631770998 2024-11-15T00:49:33,631 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c4e3e3ea69cb47668a525cca0ac2b7a7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731631772188 2024-11-15T00:49:33,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742085_1261 (size=12949) 2024-11-15T00:49:33,644 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#C#compaction#219 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:33,644 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/556e9db4d01f4908b82fa340bcc89e81 is 50, key is test_row_0/C:col10/1731631772188/Put/seqid=0 2024-11-15T00:49:33,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742086_1262 (size=12949) 2024-11-15T00:49:33,691 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/556e9db4d01f4908b82fa340bcc89e81 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/556e9db4d01f4908b82fa340bcc89e81 2024-11-15T00:49:33,701 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/C of 37711fc54bf4a3c70220e87c7201b430 into 556e9db4d01f4908b82fa340bcc89e81(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:33,701 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:33,701 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/C, priority=12, startTime=1731631773583; duration=0sec 2024-11-15T00:49:33,701 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:33,701 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:C 2024-11-15T00:49:33,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-15T00:49:33,721 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:33,721 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T00:49:33,728 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7806c1f81f0a465d817baa7d405ac9be is 50, key is test_row_0/A:col10/1731631773398/Put/seqid=0 2024-11-15T00:49:33,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:33,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:33,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742087_1263 (size=12301) 2024-11-15T00:49:33,759 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7806c1f81f0a465d817baa7d405ac9be 2024-11-15T00:49:33,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/ba072df5cec24e13b5c4231a05545684 is 50, key is test_row_0/B:col10/1731631773398/Put/seqid=0 2024-11-15T00:49:33,783 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742088_1264 (size=12301) 2024-11-15T00:49:33,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631833824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:33,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:33,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631833824, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,828 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T00:49:33,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631833937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:33,940 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631833937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:33,940 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:33,941 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:34,048 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/95869ff8b6174c7c8a822a87b4fef49a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/95869ff8b6174c7c8a822a87b4fef49a 2024-11-15T00:49:34,053 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/A of 37711fc54bf4a3c70220e87c7201b430 into 95869ff8b6174c7c8a822a87b4fef49a(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:34,053 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:34,053 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/A, priority=12, startTime=1731631773583; duration=0sec 2024-11-15T00:49:34,053 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:34,053 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:A 2024-11-15T00:49:34,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631834147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:34,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631834147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:34,184 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/ba072df5cec24e13b5c4231a05545684 2024-11-15T00:49:34,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/e2bf5479bfd044cf8c64e5bf511ede63 is 50, key is test_row_0/C:col10/1731631773398/Put/seqid=0 2024-11-15T00:49:34,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742089_1265 (size=12301) 2024-11-15T00:49:34,233 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=301 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/e2bf5479bfd044cf8c64e5bf511ede63 2024-11-15T00:49:34,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/7806c1f81f0a465d817baa7d405ac9be as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7806c1f81f0a465d817baa7d405ac9be 2024-11-15T00:49:34,243 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7806c1f81f0a465d817baa7d405ac9be, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T00:49:34,245 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/ba072df5cec24e13b5c4231a05545684 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ba072df5cec24e13b5c4231a05545684 2024-11-15T00:49:34,250 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ba072df5cec24e13b5c4231a05545684, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T00:49:34,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/e2bf5479bfd044cf8c64e5bf511ede63 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2bf5479bfd044cf8c64e5bf511ede63 2024-11-15T00:49:34,275 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2bf5479bfd044cf8c64e5bf511ede63, entries=150, sequenceid=301, filesize=12.0 K 2024-11-15T00:49:34,279 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 37711fc54bf4a3c70220e87c7201b430 in 558ms, sequenceid=301, compaction requested=false 2024-11-15T00:49:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:34,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-15T00:49:34,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-15T00:49:34,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-15T00:49:34,285 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0230 sec 2024-11-15T00:49:34,288 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.0290 sec 2024-11-15T00:49:34,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-15T00:49:34,397 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:34,397 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:34,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-15T00:49:34,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T00:49:34,399 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:34,400 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:34,400 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:34,461 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:34,461 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T00:49:34,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:34,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/3130ecb8f0f2432fbb9edebc2ba7deec is 50, key is test_row_0/A:col10/1731631773822/Put/seqid=0 2024-11-15T00:49:34,497 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:34,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631834493, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,498 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:34,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742090_1266 (size=17181) 2024-11-15T00:49:34,500 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:34,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631834498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:34,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/3130ecb8f0f2432fbb9edebc2ba7deec 2024-11-15T00:49:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,501 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:34,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T00:49:34,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/0ab2d3bc1f6448eb9a0a62fb0229b4a9 is 50, key is test_row_0/B:col10/1731631773822/Put/seqid=0 2024-11-15T00:49:34,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T00:49:34,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:34,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:34,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:34,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:34,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:34,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:34,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742091_1267 (size=12301) 2024-11-15T00:49:34,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:34,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631834607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:34,611 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:34,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631834607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,612 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:34,706 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T00:49:34,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:34,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:34,706 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:34,707 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:34,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:34,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:34,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T00:49:34,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:34,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631834821, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:34,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:34,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631834822, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:34,836 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:34,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T00:49:34,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:34,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:34,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:34,876 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:34,876 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:34,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:34,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/0ab2d3bc1f6448eb9a0a62fb0229b4a9 2024-11-15T00:49:34,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/b2019b8f8e4f4114861dfa92d2f4d46c is 50, key is test_row_0/C:col10/1731631773822/Put/seqid=0 2024-11-15T00:49:35,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742092_1268 (size=12301) 2024-11-15T00:49:35,015 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/b2019b8f8e4f4114861dfa92d2f4d46c 2024-11-15T00:49:35,025 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/3130ecb8f0f2432fbb9edebc2ba7deec as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3130ecb8f0f2432fbb9edebc2ba7deec 2024-11-15T00:49:35,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T00:49:35,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T00:49:35,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:35,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,032 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,035 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3130ecb8f0f2432fbb9edebc2ba7deec, entries=250, sequenceid=325, filesize=16.8 K 2024-11-15T00:49:35,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/0ab2d3bc1f6448eb9a0a62fb0229b4a9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0ab2d3bc1f6448eb9a0a62fb0229b4a9 2024-11-15T00:49:35,046 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0ab2d3bc1f6448eb9a0a62fb0229b4a9, entries=150, sequenceid=325, filesize=12.0 K 2024-11-15T00:49:35,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/b2019b8f8e4f4114861dfa92d2f4d46c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b2019b8f8e4f4114861dfa92d2f4d46c 2024-11-15T00:49:35,087 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b2019b8f8e4f4114861dfa92d2f4d46c, entries=150, sequenceid=325, filesize=12.0 K 2024-11-15T00:49:35,088 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 37711fc54bf4a3c70220e87c7201b430 in 627ms, sequenceid=325, compaction requested=true 2024-11-15T00:49:35,088 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:35,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:35,088 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:35,088 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:35,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:35,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:35,089 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:35,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:35,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:35,090 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:35,090 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 42431 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:35,090 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/A is initiating minor compaction (all files) 2024-11-15T00:49:35,090 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/B is initiating minor compaction (all files) 2024-11-15T00:49:35,090 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/B in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,090 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/A in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,090 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ae3afd73f5ff40ec8fc1d89d553a1bbb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ba072df5cec24e13b5c4231a05545684, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0ab2d3bc1f6448eb9a0a62fb0229b4a9] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=36.7 K 2024-11-15T00:49:35,090 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/95869ff8b6174c7c8a822a87b4fef49a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7806c1f81f0a465d817baa7d405ac9be, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3130ecb8f0f2432fbb9edebc2ba7deec] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=41.4 K 2024-11-15T00:49:35,090 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95869ff8b6174c7c8a822a87b4fef49a, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731631772188 2024-11-15T00:49:35,091 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7806c1f81f0a465d817baa7d405ac9be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731631773398 2024-11-15T00:49:35,091 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3130ecb8f0f2432fbb9edebc2ba7deec, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631773818 2024-11-15T00:49:35,092 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ae3afd73f5ff40ec8fc1d89d553a1bbb, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731631772188 2024-11-15T00:49:35,093 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ba072df5cec24e13b5c4231a05545684, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731631773398 2024-11-15T00:49:35,093 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0ab2d3bc1f6448eb9a0a62fb0229b4a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631773818 2024-11-15T00:49:35,122 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#A#compaction#226 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:35,123 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/b559fbba9f7e4288950560679da3b354 is 50, key is test_row_0/A:col10/1731631773822/Put/seqid=0 2024-11-15T00:49:35,130 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#B#compaction#227 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:35,130 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/715c7643bd5947589d6d03f77107c7da is 50, key is test_row_0/B:col10/1731631773822/Put/seqid=0 2024-11-15T00:49:35,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:35,150 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:35,150 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T00:49:35,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742093_1269 (size=13051) 2024-11-15T00:49:35,162 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/649ab937c1184d9ab57889fa87605cca is 50, key is test_row_0/A:col10/1731631774493/Put/seqid=0 2024-11-15T00:49:35,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742094_1270 (size=13051) 2024-11-15T00:49:35,175 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/b559fbba9f7e4288950560679da3b354 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b559fbba9f7e4288950560679da3b354 2024-11-15T00:49:35,179 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742095_1271 (size=14741) 2024-11-15T00:49:35,179 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/649ab937c1184d9ab57889fa87605cca 2024-11-15T00:49:35,181 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/A of 37711fc54bf4a3c70220e87c7201b430 into b559fbba9f7e4288950560679da3b354(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:35,181 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:35,181 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/A, priority=13, startTime=1731631775088; duration=0sec 2024-11-15T00:49:35,181 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:35,181 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:A 2024-11-15T00:49:35,181 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:35,182 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:35,182 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/C is initiating minor compaction (all files) 2024-11-15T00:49:35,182 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/C in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,183 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/556e9db4d01f4908b82fa340bcc89e81, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2bf5479bfd044cf8c64e5bf511ede63, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b2019b8f8e4f4114861dfa92d2f4d46c] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=36.7 K 2024-11-15T00:49:35,183 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 556e9db4d01f4908b82fa340bcc89e81, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1731631772188 2024-11-15T00:49:35,183 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e2bf5479bfd044cf8c64e5bf511ede63, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=301, earliestPutTs=1731631773398 2024-11-15T00:49:35,184 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b2019b8f8e4f4114861dfa92d2f4d46c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631773818 2024-11-15T00:49:35,185 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T00:49:35,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:35,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,185 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,185 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,189 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/e2a95126bab440c48bf72dec5315c795 is 50, key is test_row_0/B:col10/1731631774493/Put/seqid=0 2024-11-15T00:49:35,193 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742096_1272 (size=12301) 2024-11-15T00:49:35,194 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#C#compaction#230 average throughput is unlimited, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:35,194 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/b364dd9f4ffb496693619a2fb69654a5 is 50, key is test_row_0/C:col10/1731631773822/Put/seqid=0 2024-11-15T00:49:35,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/e2a95126bab440c48bf72dec5315c795 2024-11-15T00:49:35,201 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742097_1273 (size=13051) 2024-11-15T00:49:35,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/4dc95dbc3661489fb022bc84e4efa184 is 50, key is test_row_0/C:col10/1731631774493/Put/seqid=0 2024-11-15T00:49:35,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631835204, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,210 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38614 deadline: 1731631835207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,210 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 37711fc54bf4a3c70220e87c7201b430 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,211 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631835209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,213 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,213 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 154 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631835209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,214 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742098_1274 (size=12301) 2024-11-15T00:49:35,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38620 deadline: 1731631835217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,222 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 37711fc54bf4a3c70220e87c7201b430 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,320 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631835320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,321 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,321 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631835320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,322 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 155 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631835320, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,322 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,339 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T00:49:35,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:35,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,339 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,491 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T00:49:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 156 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631835526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631835526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631835526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,528 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,529 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T00:49:35,576 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/715c7643bd5947589d6d03f77107c7da as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/715c7643bd5947589d6d03f77107c7da 2024-11-15T00:49:35,585 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/B of 37711fc54bf4a3c70220e87c7201b430 into 715c7643bd5947589d6d03f77107c7da(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:35,585 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:35,585 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/B, priority=13, startTime=1731631775088; duration=0sec 2024-11-15T00:49:35,585 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:35,585 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:B 2024-11-15T00:49:35,609 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/b364dd9f4ffb496693619a2fb69654a5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b364dd9f4ffb496693619a2fb69654a5 2024-11-15T00:49:35,616 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/C of 37711fc54bf4a3c70220e87c7201b430 into b364dd9f4ffb496693619a2fb69654a5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:35,616 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:35,616 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/C, priority=13, startTime=1731631775089; duration=0sec 2024-11-15T00:49:35,616 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:35,617 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:C 2024-11-15T00:49:35,618 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=340 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/4dc95dbc3661489fb022bc84e4efa184 2024-11-15T00:49:35,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/649ab937c1184d9ab57889fa87605cca as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/649ab937c1184d9ab57889fa87605cca 2024-11-15T00:49:35,633 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/649ab937c1184d9ab57889fa87605cca, entries=200, sequenceid=340, filesize=14.4 K 2024-11-15T00:49:35,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/e2a95126bab440c48bf72dec5315c795 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e2a95126bab440c48bf72dec5315c795 2024-11-15T00:49:35,652 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e2a95126bab440c48bf72dec5315c795, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T00:49:35,652 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T00:49:35,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:35,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,653 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,654 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/4dc95dbc3661489fb022bc84e4efa184 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4dc95dbc3661489fb022bc84e4efa184 2024-11-15T00:49:35,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:35,677 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4dc95dbc3661489fb022bc84e4efa184, entries=150, sequenceid=340, filesize=12.0 K 2024-11-15T00:49:35,678 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 37711fc54bf4a3c70220e87c7201b430 in 528ms, sequenceid=340, compaction requested=false 2024-11-15T00:49:35,678 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:35,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-15T00:49:35,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:35,824 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T00:49:35,830 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/fdc6d1ff13ba493b8de6d939a4771dab is 50, key is test_row_0/A:col10/1731631775206/Put/seqid=0 2024-11-15T00:49:35,833 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742099_1275 (size=12301) 2024-11-15T00:49:35,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:35,838 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. as already flushing 2024-11-15T00:49:35,853 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631835849, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,855 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631835854, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,860 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,861 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631835857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,862 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,959 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631835956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631835967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631835967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,969 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:35,970 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:36,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:36,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631836169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:36,182 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:36,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631836178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:36,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631836178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,183 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:36,235 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/fdc6d1ff13ba493b8de6d939a4771dab 2024-11-15T00:49:36,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/64aa38ab64e64358a162fed51574dcc3 is 50, key is test_row_0/B:col10/1731631775206/Put/seqid=0 2024-11-15T00:49:36,245 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742100_1276 (size=12301) 2024-11-15T00:49:36,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631836486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:36,488 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:36,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631836486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,488 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:36,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631836486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:36,489 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:36,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T00:49:36,646 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/64aa38ab64e64358a162fed51574dcc3 2024-11-15T00:49:36,653 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/a54ffa5001a946d5b0eadbec36978c4c is 50, key is test_row_0/C:col10/1731631775206/Put/seqid=0 2024-11-15T00:49:36,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742101_1277 (size=12301) 2024-11-15T00:49:37,000 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:37,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38554 deadline: 1731631836996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:37,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38564 deadline: 1731631836997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:37,000 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:37,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38536 deadline: 1731631836997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:37,000 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:37,002 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:49:37,056 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=365 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/a54ffa5001a946d5b0eadbec36978c4c 2024-11-15T00:49:37,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/fdc6d1ff13ba493b8de6d939a4771dab as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/fdc6d1ff13ba493b8de6d939a4771dab 2024-11-15T00:49:37,073 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/fdc6d1ff13ba493b8de6d939a4771dab, entries=150, sequenceid=365, filesize=12.0 K 2024-11-15T00:49:37,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/64aa38ab64e64358a162fed51574dcc3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/64aa38ab64e64358a162fed51574dcc3 2024-11-15T00:49:37,081 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/64aa38ab64e64358a162fed51574dcc3, entries=150, sequenceid=365, filesize=12.0 K 2024-11-15T00:49:37,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/a54ffa5001a946d5b0eadbec36978c4c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/a54ffa5001a946d5b0eadbec36978c4c 2024-11-15T00:49:37,088 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/a54ffa5001a946d5b0eadbec36978c4c, entries=150, sequenceid=365, filesize=12.0 K 2024-11-15T00:49:37,088 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 37711fc54bf4a3c70220e87c7201b430 in 1264ms, sequenceid=365, compaction requested=true 2024-11-15T00:49:37,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:37,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:37,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-15T00:49:37,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-15T00:49:37,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-15T00:49:37,095 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6920 sec 2024-11-15T00:49:37,100 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.6990 sec 2024-11-15T00:49:37,335 INFO [Thread-980 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-980. 2024-11-15T00:49:37,335 DEBUG [Thread-980 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:37,335 DEBUG [Thread-980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:37,335 DEBUG [Thread-980 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:37,336 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:37,337 INFO [Thread-982 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-982. 2024-11-15T00:49:37,337 DEBUG [Thread-982 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:37,337 DEBUG [Thread-982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:37,337 DEBUG [Thread-982 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:37,337 INFO [Thread-978 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-978. 2024-11-15T00:49:37,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:37,337 DEBUG [Thread-978 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:37,337 DEBUG [Thread-978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:37,337 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:37,337 DEBUG [Thread-978 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:37,339 INFO [Thread-976 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-976. 2024-11-15T00:49:37,339 DEBUG [Thread-976 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:37,339 DEBUG [Thread-976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:37,339 DEBUG [Thread-976 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:37,339 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:37,341 INFO [Thread-984 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-984. 2024-11-15T00:49:37,341 DEBUG [Thread-984 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:37,341 DEBUG [Thread-984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:37,341 DEBUG [Thread-984 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:37,343 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:38,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:38,008 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:38,008 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-15T00:49:38,008 INFO [Thread-971 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-971. 2024-11-15T00:49:38,008 INFO [Thread-965 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-965. 2024-11-15T00:49:38,008 DEBUG [Thread-971 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:38,008 DEBUG [Thread-965 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:38,008 DEBUG [Thread-971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:38,008 DEBUG [Thread-965 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:38,008 DEBUG [Thread-971 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:38,008 DEBUG [Thread-965 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:38,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:38,008 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:38,012 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/9327372211ae42ea85c6bcca33646c13 is 50, key is test_row_0/A:col10/1731631775851/Put/seqid=0 2024-11-15T00:49:38,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742102_1278 (size=12301) 2024-11-15T00:49:38,018 INFO [Thread-967 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-967. 2024-11-15T00:49:38,018 DEBUG [Thread-967 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:38,018 DEBUG [Thread-967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:38,018 DEBUG [Thread-967 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:38,018 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:38,416 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/9327372211ae42ea85c6bcca33646c13 2024-11-15T00:49:38,423 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/01ac36a969004566bec76961aa6945c6 is 50, key is test_row_0/B:col10/1731631775851/Put/seqid=0 2024-11-15T00:49:38,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742103_1279 (size=12301) 2024-11-15T00:49:38,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-15T00:49:38,557 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:38,830 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/01ac36a969004566bec76961aa6945c6 2024-11-15T00:49:38,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/49fc67b7439b4f9794de683dde904680 is 50, key is test_row_0/C:col10/1731631775851/Put/seqid=0 2024-11-15T00:49:38,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742104_1280 (size=12301) 2024-11-15T00:49:39,241 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=381 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/49fc67b7439b4f9794de683dde904680 2024-11-15T00:49:39,245 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/9327372211ae42ea85c6bcca33646c13 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9327372211ae42ea85c6bcca33646c13 2024-11-15T00:49:39,249 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9327372211ae42ea85c6bcca33646c13, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T00:49:39,250 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/01ac36a969004566bec76961aa6945c6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/01ac36a969004566bec76961aa6945c6 2024-11-15T00:49:39,255 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/01ac36a969004566bec76961aa6945c6, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T00:49:39,256 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/49fc67b7439b4f9794de683dde904680 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/49fc67b7439b4f9794de683dde904680 2024-11-15T00:49:39,260 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/49fc67b7439b4f9794de683dde904680, entries=150, sequenceid=381, filesize=12.0 K 2024-11-15T00:49:39,261 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=6.71 KB/6870 for 37711fc54bf4a3c70220e87c7201b430 in 1253ms, sequenceid=381, compaction requested=true 2024-11-15T00:49:39,261 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:39,261 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:39,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:39,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:39,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:39,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:39,261 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:39,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 37711fc54bf4a3c70220e87c7201b430:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:39,261 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:39,262 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:39,262 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:39,262 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/A is initiating minor compaction (all files) 2024-11-15T00:49:39,262 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/B is initiating minor compaction (all files) 2024-11-15T00:49:39,262 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/B in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:39,262 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/A in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:39,262 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/715c7643bd5947589d6d03f77107c7da, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e2a95126bab440c48bf72dec5315c795, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/64aa38ab64e64358a162fed51574dcc3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/01ac36a969004566bec76961aa6945c6] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=48.8 K 2024-11-15T00:49:39,262 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b559fbba9f7e4288950560679da3b354, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/649ab937c1184d9ab57889fa87605cca, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/fdc6d1ff13ba493b8de6d939a4771dab, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9327372211ae42ea85c6bcca33646c13] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=51.2 K 2024-11-15T00:49:39,263 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b559fbba9f7e4288950560679da3b354, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631773818 2024-11-15T00:49:39,263 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 715c7643bd5947589d6d03f77107c7da, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631773818 2024-11-15T00:49:39,263 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 649ab937c1184d9ab57889fa87605cca, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731631774484 2024-11-15T00:49:39,263 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e2a95126bab440c48bf72dec5315c795, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731631774493 2024-11-15T00:49:39,263 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdc6d1ff13ba493b8de6d939a4771dab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731631775198 2024-11-15T00:49:39,263 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 64aa38ab64e64358a162fed51574dcc3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731631775198 2024-11-15T00:49:39,263 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9327372211ae42ea85c6bcca33646c13, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731631775851 2024-11-15T00:49:39,263 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 01ac36a969004566bec76961aa6945c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731631775851 2024-11-15T00:49:39,270 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#A#compaction#238 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:39,271 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#B#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:39,271 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/4e9f672ce1094cfcba743ee92a8e8dc3 is 50, key is test_row_0/A:col10/1731631775851/Put/seqid=0 2024-11-15T00:49:39,271 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/81ee27721cc1459ea68923b352fbbe9d is 50, key is test_row_0/B:col10/1731631775851/Put/seqid=0 2024-11-15T00:49:39,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742106_1282 (size=13187) 2024-11-15T00:49:39,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742105_1281 (size=13187) 2024-11-15T00:49:39,286 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/81ee27721cc1459ea68923b352fbbe9d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/81ee27721cc1459ea68923b352fbbe9d 2024-11-15T00:49:39,286 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/4e9f672ce1094cfcba743ee92a8e8dc3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/4e9f672ce1094cfcba743ee92a8e8dc3 2024-11-15T00:49:39,291 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/A of 37711fc54bf4a3c70220e87c7201b430 into 4e9f672ce1094cfcba743ee92a8e8dc3(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:39,291 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:39,291 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/B of 37711fc54bf4a3c70220e87c7201b430 into 81ee27721cc1459ea68923b352fbbe9d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:39,291 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/A, priority=12, startTime=1731631779261; duration=0sec 2024-11-15T00:49:39,291 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:39,291 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/B, priority=12, startTime=1731631779261; duration=0sec 2024-11-15T00:49:39,291 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:39,291 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:A 2024-11-15T00:49:39,291 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:39,291 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:39,291 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:B 2024-11-15T00:49:39,292 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:39,293 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 37711fc54bf4a3c70220e87c7201b430/C is initiating minor compaction (all files) 2024-11-15T00:49:39,293 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 37711fc54bf4a3c70220e87c7201b430/C in TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:39,293 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b364dd9f4ffb496693619a2fb69654a5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4dc95dbc3661489fb022bc84e4efa184, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/a54ffa5001a946d5b0eadbec36978c4c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/49fc67b7439b4f9794de683dde904680] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp, totalSize=48.8 K 2024-11-15T00:49:39,293 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b364dd9f4ffb496693619a2fb69654a5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631773818 2024-11-15T00:49:39,293 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4dc95dbc3661489fb022bc84e4efa184, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=340, earliestPutTs=1731631774493 2024-11-15T00:49:39,294 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a54ffa5001a946d5b0eadbec36978c4c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=365, earliestPutTs=1731631775198 2024-11-15T00:49:39,294 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49fc67b7439b4f9794de683dde904680, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=381, earliestPutTs=1731631775851 2024-11-15T00:49:39,303 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 37711fc54bf4a3c70220e87c7201b430#C#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:39,304 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/4eb6f6a9ad9647b9ad1cd2274c9a0d29 is 50, key is test_row_0/C:col10/1731631775851/Put/seqid=0 2024-11-15T00:49:39,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742107_1283 (size=13187) 2024-11-15T00:49:39,712 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/4eb6f6a9ad9647b9ad1cd2274c9a0d29 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4eb6f6a9ad9647b9ad1cd2274c9a0d29 2024-11-15T00:49:39,716 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 37711fc54bf4a3c70220e87c7201b430/C of 37711fc54bf4a3c70220e87c7201b430 into 4eb6f6a9ad9647b9ad1cd2274c9a0d29(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:39,716 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 37711fc54bf4a3c70220e87c7201b430: 2024-11-15T00:49:39,716 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., storeName=37711fc54bf4a3c70220e87c7201b430/C, priority=12, startTime=1731631779261; duration=0sec 2024-11-15T00:49:39,716 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:39,716 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 37711fc54bf4a3c70220e87c7201b430:C 2024-11-15T00:49:45,238 INFO [Thread-973 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-973. 2024-11-15T00:49:45,238 DEBUG [Thread-973 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:45,238 DEBUG [Thread-973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:45,238 DEBUG [Thread-973 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:45,238 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:45,319 INFO [Thread-969 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-969. 2024-11-15T00:49:45,319 DEBUG [Thread-969 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:49:45,319 DEBUG [Thread-969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:45,319 DEBUG [Thread-969 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:45,319 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:45,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:49:45,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 102 2024-11-15T00:49:45,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 103 2024-11-15T00:49:45,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 30 2024-11-15T00:49:45,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 37 2024-11-15T00:49:45,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-15T00:49:45,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:49:45,319 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3074 2024-11-15T00:49:45,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3313 2024-11-15T00:49:45,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3057 2024-11-15T00:49:45,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3063 2024-11-15T00:49:45,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3284 2024-11-15T00:49:45,320 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:49:45,320 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:49:45,320 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:49:45,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:45,320 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:45,320 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:49:45,321 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:49:45,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:45,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T00:49:45,326 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631785326"}]},"ts":"1731631785326"} 2024-11-15T00:49:45,328 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:49:45,329 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:49:45,329 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:49:45,331 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37711fc54bf4a3c70220e87c7201b430, UNASSIGN}] 2024-11-15T00:49:45,332 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37711fc54bf4a3c70220e87c7201b430, UNASSIGN 2024-11-15T00:49:45,332 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=37711fc54bf4a3c70220e87c7201b430, regionState=CLOSING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:45,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=84, ppid=83, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=37711fc54bf4a3c70220e87c7201b430, UNASSIGN because future has completed 2024-11-15T00:49:45,334 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:49:45,334 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:49:45,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T00:49:45,486 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(122): Close 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:45,486 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:49:45,486 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1722): Closing 37711fc54bf4a3c70220e87c7201b430, disabling compactions & flushes 2024-11-15T00:49:45,486 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:45,486 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:45,486 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. after waiting 0 ms 2024-11-15T00:49:45,486 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:45,486 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(2902): Flushing 37711fc54bf4a3c70220e87c7201b430 3/3 column families, dataSize=20.13 KB heapSize=53.48 KB 2024-11-15T00:49:45,491 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/84841341c3da4c4b90b02b5a35d0a509 is 50, key is test_row_0/A:col10/1731631785317/Put/seqid=0 2024-11-15T00:49:45,494 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742108_1284 (size=7415) 2024-11-15T00:49:45,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T00:49:45,895 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/84841341c3da4c4b90b02b5a35d0a509 2024-11-15T00:49:45,909 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/833dfc2028454dbaa9b79ae8ef951e61 is 50, key is test_row_0/B:col10/1731631785317/Put/seqid=0 2024-11-15T00:49:45,912 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742109_1285 (size=7415) 2024-11-15T00:49:45,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T00:49:46,313 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/833dfc2028454dbaa9b79ae8ef951e61 2024-11-15T00:49:46,323 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/f20847e441b74c4fbda50906bd80bb1d is 50, key is test_row_0/C:col10/1731631785317/Put/seqid=0 2024-11-15T00:49:46,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742110_1286 (size=7415) 2024-11-15T00:49:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T00:49:46,727 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=6.71 KB at sequenceid=390 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/f20847e441b74c4fbda50906bd80bb1d 2024-11-15T00:49:46,733 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/A/84841341c3da4c4b90b02b5a35d0a509 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/84841341c3da4c4b90b02b5a35d0a509 2024-11-15T00:49:46,737 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/84841341c3da4c4b90b02b5a35d0a509, entries=50, sequenceid=390, filesize=7.2 K 2024-11-15T00:49:46,738 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/B/833dfc2028454dbaa9b79ae8ef951e61 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/833dfc2028454dbaa9b79ae8ef951e61 2024-11-15T00:49:46,741 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/833dfc2028454dbaa9b79ae8ef951e61, entries=50, sequenceid=390, filesize=7.2 K 2024-11-15T00:49:46,742 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/.tmp/C/f20847e441b74c4fbda50906bd80bb1d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/f20847e441b74c4fbda50906bd80bb1d 2024-11-15T00:49:46,745 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/f20847e441b74c4fbda50906bd80bb1d, entries=50, sequenceid=390, filesize=7.2 K 2024-11-15T00:49:46,746 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 37711fc54bf4a3c70220e87c7201b430 in 1260ms, sequenceid=390, compaction requested=false 2024-11-15T00:49:46,747 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/327ed577e0f24b498c5f2d1f71e57e7a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/4b82cdd2b11a43f0bf853065fbd2442d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/442e141812024ed18cd467f78eb91e16, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3b0813c8f9094ecf9f166dbc48da24c9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/43078e7b21f9486e877fef2f5620faf8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7656aa2ed33c427a81fb1cb637e04735, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/72f7ba5904af4560947e635eab7f4dd5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/23631f4bc6ae4899978ca3cf599b5591, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/5985b1859e414920ad32d97e13c01b27, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/30b69031f53e4ce1b0dc4d1232991d4d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/593d94b915364829b67c87946a0eacbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7465e83ba09a4c21aef96eb4ee0bb345, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/35b8d36006af438da67751cd34a205b8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/283f38e6d81a42e39d83a6c95d195cf4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7a764cd0cbaa4e00868045886ba90b90, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9048f928942849c5b93af3e08b780827, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/eeefd778716f4cf3b824b4d8c92e9d7e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b1581f6f278242c6bb3d838c961d444b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/f88c3d1575b94782b261d30ba88be4ef, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/95869ff8b6174c7c8a822a87b4fef49a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7806c1f81f0a465d817baa7d405ac9be, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3130ecb8f0f2432fbb9edebc2ba7deec, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b559fbba9f7e4288950560679da3b354, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/649ab937c1184d9ab57889fa87605cca, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/fdc6d1ff13ba493b8de6d939a4771dab, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9327372211ae42ea85c6bcca33646c13] to archive 2024-11-15T00:49:46,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:49:46,749 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/327ed577e0f24b498c5f2d1f71e57e7a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/327ed577e0f24b498c5f2d1f71e57e7a 2024-11-15T00:49:46,750 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/4b82cdd2b11a43f0bf853065fbd2442d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/4b82cdd2b11a43f0bf853065fbd2442d 2024-11-15T00:49:46,752 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/442e141812024ed18cd467f78eb91e16 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/442e141812024ed18cd467f78eb91e16 2024-11-15T00:49:46,753 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3b0813c8f9094ecf9f166dbc48da24c9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3b0813c8f9094ecf9f166dbc48da24c9 2024-11-15T00:49:46,754 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/43078e7b21f9486e877fef2f5620faf8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/43078e7b21f9486e877fef2f5620faf8 2024-11-15T00:49:46,755 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7656aa2ed33c427a81fb1cb637e04735 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7656aa2ed33c427a81fb1cb637e04735 2024-11-15T00:49:46,757 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/72f7ba5904af4560947e635eab7f4dd5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/72f7ba5904af4560947e635eab7f4dd5 2024-11-15T00:49:46,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/23631f4bc6ae4899978ca3cf599b5591 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/23631f4bc6ae4899978ca3cf599b5591 2024-11-15T00:49:46,761 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/5985b1859e414920ad32d97e13c01b27 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/5985b1859e414920ad32d97e13c01b27 2024-11-15T00:49:46,762 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/30b69031f53e4ce1b0dc4d1232991d4d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/30b69031f53e4ce1b0dc4d1232991d4d 2024-11-15T00:49:46,764 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/593d94b915364829b67c87946a0eacbf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/593d94b915364829b67c87946a0eacbf 2024-11-15T00:49:46,765 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7465e83ba09a4c21aef96eb4ee0bb345 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7465e83ba09a4c21aef96eb4ee0bb345 2024-11-15T00:49:46,766 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/35b8d36006af438da67751cd34a205b8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/35b8d36006af438da67751cd34a205b8 2024-11-15T00:49:46,767 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/283f38e6d81a42e39d83a6c95d195cf4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/283f38e6d81a42e39d83a6c95d195cf4 2024-11-15T00:49:46,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7a764cd0cbaa4e00868045886ba90b90 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7a764cd0cbaa4e00868045886ba90b90 2024-11-15T00:49:46,769 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9048f928942849c5b93af3e08b780827 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9048f928942849c5b93af3e08b780827 2024-11-15T00:49:46,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/eeefd778716f4cf3b824b4d8c92e9d7e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/eeefd778716f4cf3b824b4d8c92e9d7e 2024-11-15T00:49:46,770 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b1581f6f278242c6bb3d838c961d444b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b1581f6f278242c6bb3d838c961d444b 2024-11-15T00:49:46,771 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/f88c3d1575b94782b261d30ba88be4ef to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/f88c3d1575b94782b261d30ba88be4ef 2024-11-15T00:49:46,772 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/95869ff8b6174c7c8a822a87b4fef49a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/95869ff8b6174c7c8a822a87b4fef49a 2024-11-15T00:49:46,773 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7806c1f81f0a465d817baa7d405ac9be to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/7806c1f81f0a465d817baa7d405ac9be 2024-11-15T00:49:46,774 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3130ecb8f0f2432fbb9edebc2ba7deec to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/3130ecb8f0f2432fbb9edebc2ba7deec 2024-11-15T00:49:46,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b559fbba9f7e4288950560679da3b354 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/b559fbba9f7e4288950560679da3b354 2024-11-15T00:49:46,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/649ab937c1184d9ab57889fa87605cca to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/649ab937c1184d9ab57889fa87605cca 2024-11-15T00:49:46,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/fdc6d1ff13ba493b8de6d939a4771dab to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/fdc6d1ff13ba493b8de6d939a4771dab 2024-11-15T00:49:46,778 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9327372211ae42ea85c6bcca33646c13 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/9327372211ae42ea85c6bcca33646c13 2024-11-15T00:49:46,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d902ff7b9e3a489bb2ade4e3d0fd35c1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/849443a8e80741c89caa2912761d62e0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/5b08b564d77b498f802147fc22e96e0d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1813d3c137b74accae448b29d4c6d440, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/9691f31b837444d2a2e63bea194f78f6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/aa0716b23afc4c10b3f4ce701caa176d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a2f79c91e78a4371a40c2b5ea2961aa7, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a8515176c57842b1a2c48bc65352f374, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/7f2187ae8a9640168ff4c16f887aaf70, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/2239d556e8624dbf936f384f56fe2e3b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1d76b6d11dcf49eabf2777ab1dcac95b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e8316cc6ec694e4fa6ff4fbdec76a9eb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/39efdde218504eccbe29fb60323e26a5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0c04fd1ee00249cc84136e828ecd67c3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/f8c35c13f95d4a118a75eb7d9aaad9b1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/414e9cf22bca4a21ab54d6ece1271cbb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/639dc34b770d41048a9a3a001d5a428a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/13cbbc6617614c948bdbb10e0427157b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ae3afd73f5ff40ec8fc1d89d553a1bbb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d61c2b7853b447429bcf688676375244, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ba072df5cec24e13b5c4231a05545684, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/715c7643bd5947589d6d03f77107c7da, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0ab2d3bc1f6448eb9a0a62fb0229b4a9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e2a95126bab440c48bf72dec5315c795, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/64aa38ab64e64358a162fed51574dcc3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/01ac36a969004566bec76961aa6945c6] to archive 2024-11-15T00:49:46,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:49:46,781 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d902ff7b9e3a489bb2ade4e3d0fd35c1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d902ff7b9e3a489bb2ade4e3d0fd35c1 2024-11-15T00:49:46,783 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/849443a8e80741c89caa2912761d62e0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/849443a8e80741c89caa2912761d62e0 2024-11-15T00:49:46,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/5b08b564d77b498f802147fc22e96e0d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/5b08b564d77b498f802147fc22e96e0d 2024-11-15T00:49:46,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1813d3c137b74accae448b29d4c6d440 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1813d3c137b74accae448b29d4c6d440 2024-11-15T00:49:46,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/9691f31b837444d2a2e63bea194f78f6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/9691f31b837444d2a2e63bea194f78f6 2024-11-15T00:49:46,788 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:49:46,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/aa0716b23afc4c10b3f4ce701caa176d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/aa0716b23afc4c10b3f4ce701caa176d 2024-11-15T00:49:46,790 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a2f79c91e78a4371a40c2b5ea2961aa7 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a2f79c91e78a4371a40c2b5ea2961aa7 2024-11-15T00:49:46,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a8515176c57842b1a2c48bc65352f374 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/a8515176c57842b1a2c48bc65352f374 2024-11-15T00:49:46,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/7f2187ae8a9640168ff4c16f887aaf70 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/7f2187ae8a9640168ff4c16f887aaf70 2024-11-15T00:49:46,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/2239d556e8624dbf936f384f56fe2e3b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/2239d556e8624dbf936f384f56fe2e3b 2024-11-15T00:49:46,794 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1d76b6d11dcf49eabf2777ab1dcac95b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/1d76b6d11dcf49eabf2777ab1dcac95b 2024-11-15T00:49:46,795 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e8316cc6ec694e4fa6ff4fbdec76a9eb to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e8316cc6ec694e4fa6ff4fbdec76a9eb 2024-11-15T00:49:46,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/39efdde218504eccbe29fb60323e26a5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/39efdde218504eccbe29fb60323e26a5 2024-11-15T00:49:46,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0c04fd1ee00249cc84136e828ecd67c3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0c04fd1ee00249cc84136e828ecd67c3 2024-11-15T00:49:46,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/f8c35c13f95d4a118a75eb7d9aaad9b1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/f8c35c13f95d4a118a75eb7d9aaad9b1 2024-11-15T00:49:46,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/414e9cf22bca4a21ab54d6ece1271cbb to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/414e9cf22bca4a21ab54d6ece1271cbb 2024-11-15T00:49:46,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/639dc34b770d41048a9a3a001d5a428a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/639dc34b770d41048a9a3a001d5a428a 2024-11-15T00:49:46,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/13cbbc6617614c948bdbb10e0427157b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/13cbbc6617614c948bdbb10e0427157b 2024-11-15T00:49:46,802 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ae3afd73f5ff40ec8fc1d89d553a1bbb to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ae3afd73f5ff40ec8fc1d89d553a1bbb 2024-11-15T00:49:46,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d61c2b7853b447429bcf688676375244 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/d61c2b7853b447429bcf688676375244 2024-11-15T00:49:46,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ba072df5cec24e13b5c4231a05545684 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/ba072df5cec24e13b5c4231a05545684 2024-11-15T00:49:46,804 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/715c7643bd5947589d6d03f77107c7da to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/715c7643bd5947589d6d03f77107c7da 2024-11-15T00:49:46,805 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0ab2d3bc1f6448eb9a0a62fb0229b4a9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/0ab2d3bc1f6448eb9a0a62fb0229b4a9 2024-11-15T00:49:46,806 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e2a95126bab440c48bf72dec5315c795 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/e2a95126bab440c48bf72dec5315c795 2024-11-15T00:49:46,807 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/64aa38ab64e64358a162fed51574dcc3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/64aa38ab64e64358a162fed51574dcc3 2024-11-15T00:49:46,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/01ac36a969004566bec76961aa6945c6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/01ac36a969004566bec76961aa6945c6 2024-11-15T00:49:46,810 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/324b54aadb86405c83ed710dd0525fbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/90a4902ae02b4e5bbd71480c706e8eaa, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4388d8edf4064515a9d3061c74ebc1c3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/78475894e74c4bad8c6273dd929bb62a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2afa2f4f5144fa2b6888535f6fa534c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d4c0016dc658432ba0a0d217d21670f2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/bf9470962a244ae6b476398a3b5a34e5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/9236cb550a2648259afc939164113470, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d0d1152c40c6447a93507897b83e7ca7, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/207953f3a0f84922bf1f7bd4ab9bf789, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b446d26ac5d7467ba82e1af1a619bcbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/f511fcb352844bd68e33bea86d7d58ee, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2009be1dc7204c28a6b7d28d8c1f7298, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/522951a0e4bb47ce9079a185966cfcbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2da096d7c3414da5b9212b67478c3a77, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/86b1a288185b408b973b67ac6fa976b7, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2b8a5b58f1014ccca0a6c2a2a59e02d3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/ae9962223f1a4e23aa8195cac9ea77c9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/556e9db4d01f4908b82fa340bcc89e81, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/c4e3e3ea69cb47668a525cca0ac2b7a7, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2bf5479bfd044cf8c64e5bf511ede63, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b364dd9f4ffb496693619a2fb69654a5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b2019b8f8e4f4114861dfa92d2f4d46c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4dc95dbc3661489fb022bc84e4efa184, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/a54ffa5001a946d5b0eadbec36978c4c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/49fc67b7439b4f9794de683dde904680] to archive 2024-11-15T00:49:46,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:49:46,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/324b54aadb86405c83ed710dd0525fbf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/324b54aadb86405c83ed710dd0525fbf 2024-11-15T00:49:46,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/90a4902ae02b4e5bbd71480c706e8eaa to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/90a4902ae02b4e5bbd71480c706e8eaa 2024-11-15T00:49:46,814 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4388d8edf4064515a9d3061c74ebc1c3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4388d8edf4064515a9d3061c74ebc1c3 2024-11-15T00:49:46,815 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/78475894e74c4bad8c6273dd929bb62a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/78475894e74c4bad8c6273dd929bb62a 2024-11-15T00:49:46,816 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2afa2f4f5144fa2b6888535f6fa534c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2afa2f4f5144fa2b6888535f6fa534c 2024-11-15T00:49:46,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d4c0016dc658432ba0a0d217d21670f2 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d4c0016dc658432ba0a0d217d21670f2 2024-11-15T00:49:46,818 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/bf9470962a244ae6b476398a3b5a34e5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/bf9470962a244ae6b476398a3b5a34e5 2024-11-15T00:49:46,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/9236cb550a2648259afc939164113470 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/9236cb550a2648259afc939164113470 2024-11-15T00:49:46,820 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d0d1152c40c6447a93507897b83e7ca7 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/d0d1152c40c6447a93507897b83e7ca7 2024-11-15T00:49:46,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/207953f3a0f84922bf1f7bd4ab9bf789 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/207953f3a0f84922bf1f7bd4ab9bf789 2024-11-15T00:49:46,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b446d26ac5d7467ba82e1af1a619bcbf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b446d26ac5d7467ba82e1af1a619bcbf 2024-11-15T00:49:46,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/f511fcb352844bd68e33bea86d7d58ee to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/f511fcb352844bd68e33bea86d7d58ee 2024-11-15T00:49:46,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2009be1dc7204c28a6b7d28d8c1f7298 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2009be1dc7204c28a6b7d28d8c1f7298 2024-11-15T00:49:46,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/522951a0e4bb47ce9079a185966cfcbf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/522951a0e4bb47ce9079a185966cfcbf 2024-11-15T00:49:46,827 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2da096d7c3414da5b9212b67478c3a77 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2da096d7c3414da5b9212b67478c3a77 2024-11-15T00:49:46,833 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/86b1a288185b408b973b67ac6fa976b7 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/86b1a288185b408b973b67ac6fa976b7 2024-11-15T00:49:46,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2b8a5b58f1014ccca0a6c2a2a59e02d3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/2b8a5b58f1014ccca0a6c2a2a59e02d3 2024-11-15T00:49:46,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/ae9962223f1a4e23aa8195cac9ea77c9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/ae9962223f1a4e23aa8195cac9ea77c9 2024-11-15T00:49:46,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/556e9db4d01f4908b82fa340bcc89e81 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/556e9db4d01f4908b82fa340bcc89e81 2024-11-15T00:49:46,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/c4e3e3ea69cb47668a525cca0ac2b7a7 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/c4e3e3ea69cb47668a525cca0ac2b7a7 2024-11-15T00:49:46,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2bf5479bfd044cf8c64e5bf511ede63 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/e2bf5479bfd044cf8c64e5bf511ede63 2024-11-15T00:49:46,841 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b364dd9f4ffb496693619a2fb69654a5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b364dd9f4ffb496693619a2fb69654a5 2024-11-15T00:49:46,842 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b2019b8f8e4f4114861dfa92d2f4d46c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/b2019b8f8e4f4114861dfa92d2f4d46c 2024-11-15T00:49:46,843 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4dc95dbc3661489fb022bc84e4efa184 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4dc95dbc3661489fb022bc84e4efa184 2024-11-15T00:49:46,845 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/a54ffa5001a946d5b0eadbec36978c4c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/a54ffa5001a946d5b0eadbec36978c4c 2024-11-15T00:49:46,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/49fc67b7439b4f9794de683dde904680 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/49fc67b7439b4f9794de683dde904680 2024-11-15T00:49:46,850 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/recovered.edits/393.seqid, newMaxSeqId=393, maxSeqId=1 2024-11-15T00:49:46,851 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. 2024-11-15T00:49:46,851 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] regionserver.HRegion(1676): Region close journal for 37711fc54bf4a3c70220e87c7201b430: Waiting for close lock at 1731631785486Running coprocessor pre-close hooks at 1731631785486Disabling compacts and flushes for region at 1731631785486Disabling writes for close at 1731631785486Obtaining lock to block concurrent updates at 1731631785486Preparing flush snapshotting stores in 37711fc54bf4a3c70220e87c7201b430 at 1731631785486Finished memstore snapshotting TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430., syncing WAL and waiting on mvcc, flushsize=dataSize=20610, getHeapSize=54720, getOffHeapSize=0, getCellsCount=450 at 1731631785487 (+1 ms)Flushing stores of TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430. at 1731631785487Flushing 37711fc54bf4a3c70220e87c7201b430/A: creating writer at 1731631785487Flushing 37711fc54bf4a3c70220e87c7201b430/A: appending metadata at 1731631785490 (+3 ms)Flushing 37711fc54bf4a3c70220e87c7201b430/A: closing flushed file at 1731631785490Flushing 37711fc54bf4a3c70220e87c7201b430/B: creating writer at 1731631785903 (+413 ms)Flushing 37711fc54bf4a3c70220e87c7201b430/B: appending metadata at 1731631785908 (+5 ms)Flushing 37711fc54bf4a3c70220e87c7201b430/B: closing flushed file at 1731631785908Flushing 37711fc54bf4a3c70220e87c7201b430/C: creating writer at 1731631786320 (+412 ms)Flushing 37711fc54bf4a3c70220e87c7201b430/C: appending metadata at 1731631786323 (+3 ms)Flushing 37711fc54bf4a3c70220e87c7201b430/C: closing flushed file at 1731631786323Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7c4e4e22: reopening flushed file at 1731631786732 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2aba16c1: reopening flushed file at 1731631786737 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@79896a9c: reopening flushed file at 1731631786741 (+4 ms)Finished flush of dataSize ~20.13 KB/20610, heapSize ~53.44 KB/54720, currentSize=0 B/0 for 37711fc54bf4a3c70220e87c7201b430 in 1260ms, sequenceid=390, compaction requested=false at 1731631786746 (+5 ms)Writing region close event to WAL at 1731631786847 (+101 ms)Running coprocessor post-close hooks at 1731631786851 (+4 ms)Closed at 1731631786851 2024-11-15T00:49:46,853 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=85}] handler.UnassignRegionHandler(157): Closed 37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:46,854 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=84 updating hbase:meta row=37711fc54bf4a3c70220e87c7201b430, regionState=CLOSED 2024-11-15T00:49:46,858 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=85, ppid=84, state=RUNNABLE, hasLock=false; CloseRegionProcedure 37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:49:46,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-15T00:49:46,861 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; CloseRegionProcedure 37711fc54bf4a3c70220e87c7201b430, server=fb2f6cb7846a,42925,1731631698709 in 1.5250 sec 2024-11-15T00:49:46,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=84, resume processing ppid=83 2024-11-15T00:49:46,862 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, ppid=83, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=37711fc54bf4a3c70220e87c7201b430, UNASSIGN in 1.5300 sec 2024-11-15T00:49:46,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-15T00:49:46,864 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5340 sec 2024-11-15T00:49:46,865 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631786865"}]},"ts":"1731631786865"} 2024-11-15T00:49:46,866 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:49:46,866 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:49:46,868 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5460 sec 2024-11-15T00:49:47,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-15T00:49:47,477 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:47,478 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:49:47,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:47,480 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=86, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:47,481 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=86, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:47,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T00:49:47,483 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:47,485 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/recovered.edits] 2024-11-15T00:49:47,488 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/4e9f672ce1094cfcba743ee92a8e8dc3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/4e9f672ce1094cfcba743ee92a8e8dc3 2024-11-15T00:49:47,489 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/84841341c3da4c4b90b02b5a35d0a509 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/A/84841341c3da4c4b90b02b5a35d0a509 2024-11-15T00:49:47,492 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/81ee27721cc1459ea68923b352fbbe9d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/81ee27721cc1459ea68923b352fbbe9d 2024-11-15T00:49:47,493 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/833dfc2028454dbaa9b79ae8ef951e61 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/B/833dfc2028454dbaa9b79ae8ef951e61 2024-11-15T00:49:47,494 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4eb6f6a9ad9647b9ad1cd2274c9a0d29 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/4eb6f6a9ad9647b9ad1cd2274c9a0d29 2024-11-15T00:49:47,495 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/f20847e441b74c4fbda50906bd80bb1d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/C/f20847e441b74c4fbda50906bd80bb1d 2024-11-15T00:49:47,497 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/recovered.edits/393.seqid to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430/recovered.edits/393.seqid 2024-11-15T00:49:47,498 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/37711fc54bf4a3c70220e87c7201b430 2024-11-15T00:49:47,498 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:49:47,500 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=86, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:47,502 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:49:47,505 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:49:47,506 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=86, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:47,506 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:49:47,506 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631787506"}]},"ts":"9223372036854775807"} 2024-11-15T00:49:47,508 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:49:47,509 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 37711fc54bf4a3c70220e87c7201b430, NAME => 'TestAcidGuarantees,,1731631755115.37711fc54bf4a3c70220e87c7201b430.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:49:47,509 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:49:47,509 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631787509"}]},"ts":"9223372036854775807"} 2024-11-15T00:49:47,511 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:49:47,512 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=86, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:47,513 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 34 msec 2024-11-15T00:49:47,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-15T00:49:47,587 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:49:47,588 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:47,597 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238 (was 244), OpenFileDescriptor=449 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=933 (was 896) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=1759 (was 1854) 2024-11-15T00:49:47,606 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=238, OpenFileDescriptor=449, MaxFileDescriptor=1048576, SystemLoadAverage=933, ProcessCount=11, AvailableMemoryMB=1758 2024-11-15T00:49:47,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:49:47,608 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:49:47,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:47,610 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:49:47,610 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:47,610 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 87 2024-11-15T00:49:47,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T00:49:47,611 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:49:47,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742111_1287 (size=959) 2024-11-15T00:49:47,620 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd 2024-11-15T00:49:47,626 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742112_1288 (size=53) 2024-11-15T00:49:47,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T00:49:47,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T00:49:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:49:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing d7dd7a757dbb0ea573f9e0b990dd7e24, disabling compactions & flushes 2024-11-15T00:49:48,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. after waiting 0 ms 2024-11-15T00:49:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:48,028 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:48,028 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for d7dd7a757dbb0ea573f9e0b990dd7e24: Waiting for close lock at 1731631788028Disabling compacts and flushes for region at 1731631788028Disabling writes for close at 1731631788028Writing region close event to WAL at 1731631788028Closed at 1731631788028 2024-11-15T00:49:48,030 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:49:48,031 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631788030"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631788030"}]},"ts":"1731631788030"} 2024-11-15T00:49:48,033 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:49:48,034 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:49:48,035 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631788034"}]},"ts":"1731631788034"} 2024-11-15T00:49:48,037 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:49:48,037 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, ASSIGN}] 2024-11-15T00:49:48,039 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, ASSIGN 2024-11-15T00:49:48,040 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,42925,1731631698709; forceNewPlan=false, retain=false 2024-11-15T00:49:48,191 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=d7dd7a757dbb0ea573f9e0b990dd7e24, regionState=OPENING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:48,192 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, ASSIGN because future has completed 2024-11-15T00:49:48,193 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:49:48,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T00:49:48,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T00:49:48,349 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:48,349 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7752): Opening region: {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:49:48,349 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,349 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:49:48,349 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7794): checking encryption for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,349 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(7797): checking classloading for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,350 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,351 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7dd7a757dbb0ea573f9e0b990dd7e24 columnFamilyName A 2024-11-15T00:49:48,351 DEBUG [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:48,352 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(327): Store=d7dd7a757dbb0ea573f9e0b990dd7e24/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:49:48,352 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,353 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7dd7a757dbb0ea573f9e0b990dd7e24 columnFamilyName B 2024-11-15T00:49:48,353 DEBUG [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:48,354 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(327): Store=d7dd7a757dbb0ea573f9e0b990dd7e24/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:49:48,354 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,355 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7dd7a757dbb0ea573f9e0b990dd7e24 columnFamilyName C 2024-11-15T00:49:48,355 DEBUG [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:48,356 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(327): Store=d7dd7a757dbb0ea573f9e0b990dd7e24/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:49:48,356 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1038): replaying wal for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,357 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,357 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,358 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1048): stopping wal replay for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,358 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1060): Cleaning up temporary data for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,358 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:49:48,360 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1093): writing seq id for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,361 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:49:48,362 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1114): Opened d7dd7a757dbb0ea573f9e0b990dd7e24; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=68210260, jitterRate=0.016412079334259033}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:49:48,362 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:48,362 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegion(1006): Region open journal for d7dd7a757dbb0ea573f9e0b990dd7e24: Running coprocessor pre-open hook at 1731631788349Writing region info on filesystem at 1731631788349Initializing all the Stores at 1731631788350 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631788350Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631788350Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631788350Cleaning up temporary data from old regions at 1731631788358 (+8 ms)Running coprocessor post-open hooks at 1731631788362 (+4 ms)Region opened successfully at 1731631788362 2024-11-15T00:49:48,363 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., pid=89, masterSystemTime=1731631788346 2024-11-15T00:49:48,365 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:48,365 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=89}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:48,366 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=d7dd7a757dbb0ea573f9e0b990dd7e24, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:48,367 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:49:48,369 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-15T00:49:48,370 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; OpenRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 in 175 msec 2024-11-15T00:49:48,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-15T00:49:48,371 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, ASSIGN in 333 msec 2024-11-15T00:49:48,372 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:49:48,373 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631788372"}]},"ts":"1731631788372"} 2024-11-15T00:49:48,374 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:49:48,375 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=87, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:49:48,377 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 767 msec 2024-11-15T00:49:48,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=87 2024-11-15T00:49:48,747 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:48,748 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@318a16b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:48,748 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:48,748 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:48,750 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:48,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:48,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:48,750 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@44c4a4a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:48,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:48,752 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43268, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:48,753 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64432f7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:48,754 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:48,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:48,756 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38002, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:48,757 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:49:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,46397,1731631697729 2024-11-15T00:49:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4cd4020 2024-11-15T00:49:48,758 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:49:48,759 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43284, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:49:48,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:49:48,761 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:49:48,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T00:49:48,780 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742113_1289 (size=995) 2024-11-15T00:49:49,178 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T00:49:49,178 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T00:49:49,181 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=91, ppid=90, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:49:49,184 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, REOPEN/MOVE}] 2024-11-15T00:49:49,185 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, REOPEN/MOVE 2024-11-15T00:49:49,185 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d7dd7a757dbb0ea573f9e0b990dd7e24, regionState=CLOSING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,187 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, REOPEN/MOVE because future has completed 2024-11-15T00:49:49,187 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:49:49,187 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:49:49,339 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(122): Close d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,339 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:49:49,339 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1722): Closing d7dd7a757dbb0ea573f9e0b990dd7e24, disabling compactions & flushes 2024-11-15T00:49:49,339 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:49,339 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:49,339 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. after waiting 0 ms 2024-11-15T00:49:49,339 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:49,343 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:49:49,343 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:49,343 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegion(1676): Region close journal for d7dd7a757dbb0ea573f9e0b990dd7e24: Waiting for close lock at 1731631789339Running coprocessor pre-close hooks at 1731631789339Disabling compacts and flushes for region at 1731631789339Disabling writes for close at 1731631789339Writing region close event to WAL at 1731631789340 (+1 ms)Running coprocessor post-close hooks at 1731631789343 (+3 ms)Closed at 1731631789343 2024-11-15T00:49:49,343 WARN [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] regionserver.HRegionServer(3299): Not adding moved region record: d7dd7a757dbb0ea573f9e0b990dd7e24 to self. 2024-11-15T00:49:49,345 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=93}] handler.UnassignRegionHandler(157): Closed d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,346 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d7dd7a757dbb0ea573f9e0b990dd7e24, regionState=CLOSED 2024-11-15T00:49:49,347 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:49:49,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-15T00:49:49,349 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; CloseRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 in 161 msec 2024-11-15T00:49:49,350 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, REOPEN/MOVE; state=CLOSED, location=fb2f6cb7846a,42925,1731631698709; forceNewPlan=false, retain=true 2024-11-15T00:49:49,501 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d7dd7a757dbb0ea573f9e0b990dd7e24, regionState=OPENING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,502 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, REOPEN/MOVE because future has completed 2024-11-15T00:49:49,502 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:49:49,660 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:49,660 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7752): Opening region: {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:49:49,661 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,661 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:49:49,661 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7794): checking encryption for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,662 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(7797): checking classloading for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,664 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,665 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7dd7a757dbb0ea573f9e0b990dd7e24 columnFamilyName A 2024-11-15T00:49:49,667 DEBUG [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:49,667 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(327): Store=d7dd7a757dbb0ea573f9e0b990dd7e24/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:49:49,668 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,668 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7dd7a757dbb0ea573f9e0b990dd7e24 columnFamilyName B 2024-11-15T00:49:49,668 DEBUG [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:49,669 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(327): Store=d7dd7a757dbb0ea573f9e0b990dd7e24/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:49:49,669 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,669 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region d7dd7a757dbb0ea573f9e0b990dd7e24 columnFamilyName C 2024-11-15T00:49:49,669 DEBUG [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:49,670 INFO [StoreOpener-d7dd7a757dbb0ea573f9e0b990dd7e24-1 {}] regionserver.HStore(327): Store=d7dd7a757dbb0ea573f9e0b990dd7e24/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:49:49,670 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1038): replaying wal for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,671 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,671 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,672 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1048): stopping wal replay for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,672 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1060): Cleaning up temporary data for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,673 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:49:49,674 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1093): writing seq id for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,675 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1114): Opened d7dd7a757dbb0ea573f9e0b990dd7e24; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59202047, jitterRate=-0.11782075464725494}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:49:49,675 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1122): Running coprocessor post-open hooks for d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,676 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegion(1006): Region open journal for d7dd7a757dbb0ea573f9e0b990dd7e24: Running coprocessor pre-open hook at 1731631789662Writing region info on filesystem at 1731631789662Initializing all the Stores at 1731631789663 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631789663Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631789664 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631789664Cleaning up temporary data from old regions at 1731631789672 (+8 ms)Running coprocessor post-open hooks at 1731631789675 (+3 ms)Region opened successfully at 1731631789676 (+1 ms) 2024-11-15T00:49:49,677 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., pid=94, masterSystemTime=1731631789654 2024-11-15T00:49:49,678 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:49,678 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=94}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:49,679 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=d7dd7a757dbb0ea573f9e0b990dd7e24, regionState=OPEN, openSeqNum=5, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,681 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=94, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:49:49,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=94, resume processing ppid=92 2024-11-15T00:49:49,683 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 in 180 msec 2024-11-15T00:49:49,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-15T00:49:49,685 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, REOPEN/MOVE in 500 msec 2024-11-15T00:49:49,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=91, resume processing ppid=90 2024-11-15T00:49:49,687 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, ppid=90, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 505 msec 2024-11-15T00:49:49,690 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 927 msec 2024-11-15T00:49:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-15T00:49:49,691 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:49,692 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64955dcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,692 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:49,692 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:49,693 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:49,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:49,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:49,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2b35f1a0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:49,693 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:49,694 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:49,694 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:49,695 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@314070cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,696 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b81bdd1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,696 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:49,697 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:49,697 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:49,698 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31c75d58, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:49,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:49,700 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43314, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:49,701 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3161c30a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,704 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3548d87c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,704 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:49,705 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:49,705 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:49,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@668035c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:49,706 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:49,707 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43336, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:49,708 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26e29585, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,708 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b60a361, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,708 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:49,709 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:49,709 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a917bf8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:49,710 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:49,711 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43352, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:49,712 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1155bb93, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,712 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f0b8a36, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,713 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:49,713 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:49,713 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:49,713 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@430ce307, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:49,714 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:49,715 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:49,715 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6fe6d8b3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,716 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4e8e9c8e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,716 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:49,716 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:49,717 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b715658, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,717 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:49,718 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:49,719 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43386, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:49,719 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5743d0d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,720 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a91e16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,720 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:49,720 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:49,721 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4fa6ca21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:49,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:49,722 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:49,722 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:49,723 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@633d91ec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,724 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@aa15fd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,724 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:49,724 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:49,725 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22780896, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:49,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:49,726 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43432, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:49,727 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4296d458, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@32b68c22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:49,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:49,729 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77ca130c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:49,729 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:49,730 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:49,730 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43448, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:49,731 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e5e8450, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,732 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@390f707c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,732 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:49:49,732 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:49:49,733 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:49:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:49:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:49:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5135cac, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,733 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:49:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:49:49,734 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:49:49,735 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:43452, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:49:49,735 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@26347409, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:49:49,736 DEBUG [Thread-1355 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:49,736 DEBUG [Thread-1359 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:49,736 DEBUG [Thread-1353 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:49,736 DEBUG [Thread-1350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:49,736 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:49,736 DEBUG [Thread-1344 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:49,737 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees 2024-11-15T00:49:49,738 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:49,738 DEBUG [Thread-1348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:49,738 DEBUG [Thread-1346 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:49,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:49,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38022, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:49,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38018, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:49,739 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:49,739 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:49,739 DEBUG [Thread-1342 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:49,739 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:49,740 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:49,740 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:49,740 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=95, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:49,740 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:49,741 DEBUG [Thread-1342 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:49,741 DEBUG [Thread-1357 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:49:49,741 DEBUG [Thread-1342 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:49,741 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38058, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:49,741 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:49,742 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:49:49,743 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38074, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:49:49,744 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:49,744 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:49:49,745 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:49:49,746 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:49:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:49:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:49:49,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:49:49,749 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:49,748 DEBUG [Thread-1361 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:49:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:49:49,752 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:49:49,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:38104, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:49:49,753 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:49:49,755 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:49:49,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:49,757 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:49,757 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:49:49,808 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115046c74685f764f088cdf9628a6b3a349_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_1/A:col10/1731631789757/Put/seqid=0 2024-11-15T00:49:49,812 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631849799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,813 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:49,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742114_1290 (size=12154) 2024-11-15T00:49:49,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631849806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,816 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:49,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631849810, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:49,816 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,817 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:49,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631849813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,823 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631849813, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,824 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:49:49,894 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T00:49:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:49,894 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:49,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:49,920 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631849916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,921 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:49,927 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631849927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,927 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:49,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631849927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631849927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,932 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:49,943 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631849937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:49,944 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,047 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T00:49:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:50,047 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:50,047 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:50,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:50,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:50,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:49:50,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631850126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,131 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631850137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631850137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631850137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,139 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,152 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631850147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,152 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T00:49:50,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:50,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:50,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:50,202 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] handler.RSProcedureHandler(58): pid=96 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:50,202 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=96 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:50,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=96 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:50,216 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:50,227 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115046c74685f764f088cdf9628a6b3a349_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115046c74685f764f088cdf9628a6b3a349_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:50,228 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/181bbb6d89ca447abc6b6f5c5120f65b, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:50,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/181bbb6d89ca447abc6b6f5c5120f65b is 175, key is test_row_1/A:col10/1731631789757/Put/seqid=0 2024-11-15T00:49:50,232 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742115_1291 (size=30951) 2024-11-15T00:49:50,232 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/181bbb6d89ca447abc6b6f5c5120f65b 2024-11-15T00:49:50,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/db340441f63f4f5ea916db457ae9cf3d is 50, key is test_row_1/B:col10/1731631789757/Put/seqid=0 2024-11-15T00:49:50,269 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742116_1292 (size=9657) 2024-11-15T00:49:50,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/db340441f63f4f5ea916db457ae9cf3d 2024-11-15T00:49:50,298 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/0838651b3de04ecbb746dff0c8db3164 is 50, key is test_row_1/C:col10/1731631789757/Put/seqid=0 2024-11-15T00:49:50,305 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742117_1293 (size=9657) 2024-11-15T00:49:50,306 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/0838651b3de04ecbb746dff0c8db3164 2024-11-15T00:49:50,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/181bbb6d89ca447abc6b6f5c5120f65b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/181bbb6d89ca447abc6b6f5c5120f65b 2024-11-15T00:49:50,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/181bbb6d89ca447abc6b6f5c5120f65b, entries=150, sequenceid=16, filesize=30.2 K 2024-11-15T00:49:50,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/db340441f63f4f5ea916db457ae9cf3d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/db340441f63f4f5ea916db457ae9cf3d 2024-11-15T00:49:50,322 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/db340441f63f4f5ea916db457ae9cf3d, entries=100, sequenceid=16, filesize=9.4 K 2024-11-15T00:49:50,323 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/0838651b3de04ecbb746dff0c8db3164 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/0838651b3de04ecbb746dff0c8db3164 2024-11-15T00:49:50,327 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/0838651b3de04ecbb746dff0c8db3164, entries=100, sequenceid=16, filesize=9.4 K 2024-11-15T00:49:50,328 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 571ms, sequenceid=16, compaction requested=false 2024-11-15T00:49:50,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:50,359 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=96 2024-11-15T00:49:50,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:50,360 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:49:50,367 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ccec9c83bc5d419e9d3b15fb48bacc5c_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631789809/Put/seqid=0 2024-11-15T00:49:50,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742118_1294 (size=12154) 2024-11-15T00:49:50,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:49:50,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:50,382 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ccec9c83bc5d419e9d3b15fb48bacc5c_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ccec9c83bc5d419e9d3b15fb48bacc5c_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:50,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f924e718f82a4822b0efa079cf2e9b56, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:50,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f924e718f82a4822b0efa079cf2e9b56 is 175, key is test_row_0/A:col10/1731631789809/Put/seqid=0 2024-11-15T00:49:50,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742119_1295 (size=30955) 2024-11-15T00:49:50,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:50,449 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:50,467 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631850457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631850460, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,469 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631850463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,472 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631850466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,473 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631850467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,474 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,585 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631850577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631850577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,586 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631850577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,587 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631850578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,587 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631850798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631850800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631850800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,804 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631850800, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,804 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,805 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:50,827 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f924e718f82a4822b0efa079cf2e9b56 2024-11-15T00:49:50,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/97350c9869434980adec47f3d18b4b96 is 50, key is test_row_0/B:col10/1731631789809/Put/seqid=0 2024-11-15T00:49:50,880 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742120_1296 (size=12001) 2024-11-15T00:49:50,883 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/97350c9869434980adec47f3d18b4b96 2024-11-15T00:49:50,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:49:50,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a2d0fe0e3ec64ec296021ff85f97bfe0 is 50, key is test_row_0/C:col10/1731631789809/Put/seqid=0 2024-11-15T00:49:50,914 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742121_1297 (size=12001) 2024-11-15T00:49:50,980 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:50,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631850977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:50,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:50,981 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631851107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,113 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631851113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,122 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631851117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,122 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,123 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631851120, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,136 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,315 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a2d0fe0e3ec64ec296021ff85f97bfe0 2024-11-15T00:49:51,322 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f924e718f82a4822b0efa079cf2e9b56 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f924e718f82a4822b0efa079cf2e9b56 2024-11-15T00:49:51,347 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f924e718f82a4822b0efa079cf2e9b56, entries=150, sequenceid=41, filesize=30.2 K 2024-11-15T00:49:51,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/97350c9869434980adec47f3d18b4b96 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/97350c9869434980adec47f3d18b4b96 2024-11-15T00:49:51,358 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/97350c9869434980adec47f3d18b4b96, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T00:49:51,360 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a2d0fe0e3ec64ec296021ff85f97bfe0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2d0fe0e3ec64ec296021ff85f97bfe0 2024-11-15T00:49:51,373 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2d0fe0e3ec64ec296021ff85f97bfe0, entries=150, sequenceid=41, filesize=11.7 K 2024-11-15T00:49:51,375 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1015ms, sequenceid=41, compaction requested=false 2024-11-15T00:49:51,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:51,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:51,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=96}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=96 2024-11-15T00:49:51,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=96 2024-11-15T00:49:51,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-15T00:49:51,384 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6420 sec 2024-11-15T00:49:51,386 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=95, table=TestAcidGuarantees in 1.6490 sec 2024-11-15T00:49:51,636 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:51,636 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T00:49:51,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:51,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153183449abeb246b391a9695f5098c6b2_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631791634/Put/seqid=0 2024-11-15T00:49:51,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742122_1298 (size=14594) 2024-11-15T00:49:51,774 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631851764, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631851765, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,775 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631851766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,775 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631851774, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,776 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,885 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631851876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,885 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,886 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631851887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631851887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,892 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,896 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631851887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,896 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:51,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=95 2024-11-15T00:49:51,897 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:51,897 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:51,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees 2024-11-15T00:49:51,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:49:51,899 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:51,900 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=97, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:51,900 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=97, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:51,993 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:51,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631851987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:51,994 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:51,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:49:52,052 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T00:49:52,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:52,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,053 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,053 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,084 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:52,101 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153183449abeb246b391a9695f5098c6b2_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153183449abeb246b391a9695f5098c6b2_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:52,108 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f44a633391e44903ad5514db925b8ea6, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:52,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,110 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f44a633391e44903ad5514db925b8ea6 is 175, key is test_row_0/A:col10/1731631791634/Put/seqid=0 2024-11-15T00:49:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631852098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,108 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631852097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631852098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631852107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,114 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742123_1299 (size=39549) 2024-11-15T00:49:52,207 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T00:49:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:52,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,207 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,208 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:49:52,362 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T00:49:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,363 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,363 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,383 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:49:52,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631852420, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631852421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631852421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631852421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,447 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,448 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,517 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T00:49:52,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:52,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,518 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:49:52,549 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=53, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f44a633391e44903ad5514db925b8ea6 2024-11-15T00:49:52,562 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/1c3ff85f555f4c2fa994e179fe2e0b38 is 50, key is test_row_0/B:col10/1731631791634/Put/seqid=0 2024-11-15T00:49:52,584 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742124_1300 (size=12001) 2024-11-15T00:49:52,677 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T00:49:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,677 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,677 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,835 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T00:49:52,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:52,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,835 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,835 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,975 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631852959, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631852960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,976 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,976 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631852960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631852960, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:52,977 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:52,988 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/1c3ff85f555f4c2fa994e179fe2e0b38 2024-11-15T00:49:52,988 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T00:49:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:52,988 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] handler.RSProcedureHandler(58): pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,988 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=98 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:52,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=98 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:53,010 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/08b080c024cc49a08af8ac670a8e7238 is 50, key is test_row_0/C:col10/1731631791634/Put/seqid=0 2024-11-15T00:49:53,045 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742125_1301 (size=12001) 2024-11-15T00:49:53,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:49:53,047 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/08b080c024cc49a08af8ac670a8e7238 2024-11-15T00:49:53,056 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f44a633391e44903ad5514db925b8ea6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f44a633391e44903ad5514db925b8ea6 2024-11-15T00:49:53,061 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f44a633391e44903ad5514db925b8ea6, entries=200, sequenceid=53, filesize=38.6 K 2024-11-15T00:49:53,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/1c3ff85f555f4c2fa994e179fe2e0b38 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1c3ff85f555f4c2fa994e179fe2e0b38 2024-11-15T00:49:53,073 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1c3ff85f555f4c2fa994e179fe2e0b38, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T00:49:53,075 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/08b080c024cc49a08af8ac670a8e7238 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/08b080c024cc49a08af8ac670a8e7238 2024-11-15T00:49:53,085 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/08b080c024cc49a08af8ac670a8e7238, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T00:49:53,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1453ms, sequenceid=53, compaction requested=true 2024-11-15T00:49:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:53,089 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:53,089 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:53,089 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:53,090 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101455 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:53,090 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:53,090 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/A is initiating minor compaction (all files) 2024-11-15T00:49:53,090 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/B is initiating minor compaction (all files) 2024-11-15T00:49:53,090 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/A in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:53,090 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/B in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:53,090 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/181bbb6d89ca447abc6b6f5c5120f65b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f924e718f82a4822b0efa079cf2e9b56, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f44a633391e44903ad5514db925b8ea6] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=99.1 K 2024-11-15T00:49:53,090 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/db340441f63f4f5ea916db457ae9cf3d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/97350c9869434980adec47f3d18b4b96, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1c3ff85f555f4c2fa994e179fe2e0b38] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=32.9 K 2024-11-15T00:49:53,090 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:53,090 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/181bbb6d89ca447abc6b6f5c5120f65b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f924e718f82a4822b0efa079cf2e9b56, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f44a633391e44903ad5514db925b8ea6] 2024-11-15T00:49:53,090 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 181bbb6d89ca447abc6b6f5c5120f65b, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631789753 2024-11-15T00:49:53,090 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting db340441f63f4f5ea916db457ae9cf3d, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631789753 2024-11-15T00:49:53,091 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 97350c9869434980adec47f3d18b4b96, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731631789807 2024-11-15T00:49:53,091 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f924e718f82a4822b0efa079cf2e9b56, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731631789807 2024-11-15T00:49:53,091 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1c3ff85f555f4c2fa994e179fe2e0b38, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631790451 2024-11-15T00:49:53,091 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f44a633391e44903ad5514db925b8ea6, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631790451 2024-11-15T00:49:53,101 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#B#compaction#253 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:53,101 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/00997875cc3647499c3c4d12dd0e4500 is 50, key is test_row_0/B:col10/1731631791634/Put/seqid=0 2024-11-15T00:49:53,117 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:53,119 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411155e28f33ea7404bbdabff8f922d8624e2_d7dd7a757dbb0ea573f9e0b990dd7e24 store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:53,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742126_1302 (size=12104) 2024-11-15T00:49:53,121 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411155e28f33ea7404bbdabff8f922d8624e2_d7dd7a757dbb0ea573f9e0b990dd7e24, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:53,121 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155e28f33ea7404bbdabff8f922d8624e2_d7dd7a757dbb0ea573f9e0b990dd7e24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:53,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=98 2024-11-15T00:49:53,141 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:53,141 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T00:49:53,147 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742127_1303 (size=4469) 2024-11-15T00:49:53,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c0e574fba8864e7b98d0f04e03f19872_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631791763/Put/seqid=0 2024-11-15T00:49:53,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742128_1304 (size=12154) 2024-11-15T00:49:53,553 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#A#compaction#254 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:53,553 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/c8b1eb5560ab4bfe8bba94efad1e4161 is 175, key is test_row_0/A:col10/1731631791634/Put/seqid=0 2024-11-15T00:49:53,556 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/00997875cc3647499c3c4d12dd0e4500 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/00997875cc3647499c3c4d12dd0e4500 2024-11-15T00:49:53,581 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/B of d7dd7a757dbb0ea573f9e0b990dd7e24 into 00997875cc3647499c3c4d12dd0e4500(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:53,581 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:53,581 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/B, priority=13, startTime=1731631793089; duration=0sec 2024-11-15T00:49:53,581 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:53,581 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:B 2024-11-15T00:49:53,581 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:53,582 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:53,582 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/C is initiating minor compaction (all files) 2024-11-15T00:49:53,582 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/C in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:53,582 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/0838651b3de04ecbb746dff0c8db3164, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2d0fe0e3ec64ec296021ff85f97bfe0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/08b080c024cc49a08af8ac670a8e7238] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=32.9 K 2024-11-15T00:49:53,584 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0838651b3de04ecbb746dff0c8db3164, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1731631789753 2024-11-15T00:49:53,592 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a2d0fe0e3ec64ec296021ff85f97bfe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1731631789807 2024-11-15T00:49:53,596 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 08b080c024cc49a08af8ac670a8e7238, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631790451 2024-11-15T00:49:53,600 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:53,604 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742129_1305 (size=31058) 2024-11-15T00:49:53,621 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c0e574fba8864e7b98d0f04e03f19872_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0e574fba8864e7b98d0f04e03f19872_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:53,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/77a86f6ff226430dad1b715da9f2bc9f, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:53,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/77a86f6ff226430dad1b715da9f2bc9f is 175, key is test_row_0/A:col10/1731631791763/Put/seqid=0 2024-11-15T00:49:53,653 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#C#compaction#256 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:53,654 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/c5d1a45b378f4f76bb678580b10468a6 is 50, key is test_row_0/C:col10/1731631791634/Put/seqid=0 2024-11-15T00:49:53,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742130_1306 (size=30955) 2024-11-15T00:49:53,710 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=77, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/77a86f6ff226430dad1b715da9f2bc9f 2024-11-15T00:49:53,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742131_1307 (size=12104) 2024-11-15T00:49:53,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/6d204828d520441fa7bffa2365059ad4 is 50, key is test_row_0/B:col10/1731631791763/Put/seqid=0 2024-11-15T00:49:53,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742132_1308 (size=12001) 2024-11-15T00:49:54,001 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:54,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:54,020 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/c8b1eb5560ab4bfe8bba94efad1e4161 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c8b1eb5560ab4bfe8bba94efad1e4161 2024-11-15T00:49:54,041 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631854027, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,051 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/A of d7dd7a757dbb0ea573f9e0b990dd7e24 into c8b1eb5560ab4bfe8bba94efad1e4161(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:54,051 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:54,051 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/A, priority=13, startTime=1731631793089; duration=0sec 2024-11-15T00:49:54,051 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:54,051 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:A 2024-11-15T00:49:54,053 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631854030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,054 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631854031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631854034, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,055 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,055 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631854037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,056 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:49:54,156 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631854147, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,157 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,161 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/c5d1a45b378f4f76bb678580b10468a6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/c5d1a45b378f4f76bb678580b10468a6 2024-11-15T00:49:54,175 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631854160, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,176 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,176 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631854167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631854167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,178 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,184 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631854174, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,185 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,196 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/C of d7dd7a757dbb0ea573f9e0b990dd7e24 into c5d1a45b378f4f76bb678580b10468a6(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:54,196 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:54,196 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/C, priority=13, startTime=1731631793089; duration=0sec 2024-11-15T00:49:54,197 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:54,197 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:C 2024-11-15T00:49:54,213 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/6d204828d520441fa7bffa2365059ad4 2024-11-15T00:49:54,230 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/90d2fac986054d4cb04c2b4bd5d9fb8f is 50, key is test_row_0/C:col10/1731631791763/Put/seqid=0 2024-11-15T00:49:54,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742133_1309 (size=12001) 2024-11-15T00:49:54,282 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=77 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/90d2fac986054d4cb04c2b4bd5d9fb8f 2024-11-15T00:49:54,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/77a86f6ff226430dad1b715da9f2bc9f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/77a86f6ff226430dad1b715da9f2bc9f 2024-11-15T00:49:54,325 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/77a86f6ff226430dad1b715da9f2bc9f, entries=150, sequenceid=77, filesize=30.2 K 2024-11-15T00:49:54,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/6d204828d520441fa7bffa2365059ad4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6d204828d520441fa7bffa2365059ad4 2024-11-15T00:49:54,334 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6d204828d520441fa7bffa2365059ad4, entries=150, sequenceid=77, filesize=11.7 K 2024-11-15T00:49:54,335 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/90d2fac986054d4cb04c2b4bd5d9fb8f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/90d2fac986054d4cb04c2b4bd5d9fb8f 2024-11-15T00:49:54,339 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/90d2fac986054d4cb04c2b4bd5d9fb8f, entries=150, sequenceid=77, filesize=11.7 K 2024-11-15T00:49:54,344 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1203ms, sequenceid=77, compaction requested=false 2024-11-15T00:49:54,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:54,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:54,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=98}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=98 2024-11-15T00:49:54,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=98 2024-11-15T00:49:54,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=97 2024-11-15T00:49:54,348 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4460 sec 2024-11-15T00:49:54,350 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=97, table=TestAcidGuarantees in 2.4520 sec 2024-11-15T00:49:54,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:54,377 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:54,377 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T00:49:54,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154cee1aad0cd747c7a1d10c36e15d4b0c_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631794031/Put/seqid=0 2024-11-15T00:49:54,424 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742134_1310 (size=12154) 2024-11-15T00:49:54,448 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631854435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631854439, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,450 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,450 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631854440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631854441, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631854448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,462 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631854560, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,573 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631854561, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,574 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,576 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631854564, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,577 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631854565, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,579 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,580 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,586 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631854568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,588 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631854777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,784 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,784 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631854777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,785 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631854787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631854787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,808 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631854799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:54,811 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:54,824 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:54,830 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411154cee1aad0cd747c7a1d10c36e15d4b0c_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154cee1aad0cd747c7a1d10c36e15d4b0c_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:54,831 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/2855952e06de4ac9af038b7bce0ced57, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:54,832 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/2855952e06de4ac9af038b7bce0ced57 is 175, key is test_row_0/A:col10/1731631794031/Put/seqid=0 2024-11-15T00:49:54,857 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742135_1311 (size=30955) 2024-11-15T00:49:55,101 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631855089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,102 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631855089, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,104 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,105 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,121 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631855108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,124 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631855113, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,124 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,128 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631855116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,145 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,258 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=93, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/2855952e06de4ac9af038b7bce0ced57 2024-11-15T00:49:55,278 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/1bf7711efd2941f69fccc58990e5ddc4 is 50, key is test_row_0/B:col10/1731631794031/Put/seqid=0 2024-11-15T00:49:55,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742136_1312 (size=12001) 2024-11-15T00:49:55,318 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/1bf7711efd2941f69fccc58990e5ddc4 2024-11-15T00:49:55,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/254f6ae5608b4321844dad0fd6c339c5 is 50, key is test_row_0/C:col10/1731631794031/Put/seqid=0 2024-11-15T00:49:55,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742137_1313 (size=12001) 2024-11-15T00:49:55,366 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=93 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/254f6ae5608b4321844dad0fd6c339c5 2024-11-15T00:49:55,374 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/2855952e06de4ac9af038b7bce0ced57 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2855952e06de4ac9af038b7bce0ced57 2024-11-15T00:49:55,381 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2855952e06de4ac9af038b7bce0ced57, entries=150, sequenceid=93, filesize=30.2 K 2024-11-15T00:49:55,382 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/1bf7711efd2941f69fccc58990e5ddc4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1bf7711efd2941f69fccc58990e5ddc4 2024-11-15T00:49:55,387 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1bf7711efd2941f69fccc58990e5ddc4, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T00:49:55,390 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/254f6ae5608b4321844dad0fd6c339c5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/254f6ae5608b4321844dad0fd6c339c5 2024-11-15T00:49:55,397 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/254f6ae5608b4321844dad0fd6c339c5, entries=150, sequenceid=93, filesize=11.7 K 2024-11-15T00:49:55,399 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1021ms, sequenceid=93, compaction requested=true 2024-11-15T00:49:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:55,399 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:55,399 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:55,400 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:55,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:55,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:55,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:55,400 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:55,401 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 92968 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:55,401 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/A is initiating minor compaction (all files) 2024-11-15T00:49:55,401 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/A in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:55,402 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c8b1eb5560ab4bfe8bba94efad1e4161, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/77a86f6ff226430dad1b715da9f2bc9f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2855952e06de4ac9af038b7bce0ced57] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=90.8 K 2024-11-15T00:49:55,402 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:55,402 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c8b1eb5560ab4bfe8bba94efad1e4161, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/77a86f6ff226430dad1b715da9f2bc9f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2855952e06de4ac9af038b7bce0ced57] 2024-11-15T00:49:55,404 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8b1eb5560ab4bfe8bba94efad1e4161, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631790451 2024-11-15T00:49:55,405 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:55,405 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/B is initiating minor compaction (all files) 2024-11-15T00:49:55,405 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/B in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:55,405 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/00997875cc3647499c3c4d12dd0e4500, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6d204828d520441fa7bffa2365059ad4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1bf7711efd2941f69fccc58990e5ddc4] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=35.3 K 2024-11-15T00:49:55,405 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 77a86f6ff226430dad1b715da9f2bc9f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731631791743 2024-11-15T00:49:55,406 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 00997875cc3647499c3c4d12dd0e4500, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631790451 2024-11-15T00:49:55,406 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2855952e06de4ac9af038b7bce0ced57, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731631794029 2024-11-15T00:49:55,406 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d204828d520441fa7bffa2365059ad4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731631791743 2024-11-15T00:49:55,406 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bf7711efd2941f69fccc58990e5ddc4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731631794029 2024-11-15T00:49:55,418 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:55,419 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#B#compaction#262 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:55,420 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/a7811d8e350f405cafc59fac5fbf3c28 is 50, key is test_row_0/B:col10/1731631794031/Put/seqid=0 2024-11-15T00:49:55,422 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115083eecc590f3406a87d25df9fd4d4d2b_d7dd7a757dbb0ea573f9e0b990dd7e24 store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:55,424 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115083eecc590f3406a87d25df9fd4d4d2b_d7dd7a757dbb0ea573f9e0b990dd7e24, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:55,424 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115083eecc590f3406a87d25df9fd4d4d2b_d7dd7a757dbb0ea573f9e0b990dd7e24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:55,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742138_1314 (size=12207) 2024-11-15T00:49:55,453 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742139_1315 (size=4469) 2024-11-15T00:49:55,457 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#A#compaction#263 average throughput is 0.63 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:55,458 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/295a26ed27a748cbbd872036a6103740 is 175, key is test_row_0/A:col10/1731631794031/Put/seqid=0 2024-11-15T00:49:55,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742140_1316 (size=31161) 2024-11-15T00:49:55,488 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/295a26ed27a748cbbd872036a6103740 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/295a26ed27a748cbbd872036a6103740 2024-11-15T00:49:55,494 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/A of d7dd7a757dbb0ea573f9e0b990dd7e24 into 295a26ed27a748cbbd872036a6103740(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:55,494 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:55,494 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/A, priority=13, startTime=1731631795399; duration=0sec 2024-11-15T00:49:55,494 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:55,494 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:A 2024-11-15T00:49:55,494 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:49:55,496 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:49:55,496 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/C is initiating minor compaction (all files) 2024-11-15T00:49:55,496 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/C in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:55,496 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/c5d1a45b378f4f76bb678580b10468a6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/90d2fac986054d4cb04c2b4bd5d9fb8f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/254f6ae5608b4321844dad0fd6c339c5] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=35.3 K 2024-11-15T00:49:55,497 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c5d1a45b378f4f76bb678580b10468a6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631790451 2024-11-15T00:49:55,497 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 90d2fac986054d4cb04c2b4bd5d9fb8f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=77, earliestPutTs=1731631791743 2024-11-15T00:49:55,497 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254f6ae5608b4321844dad0fd6c339c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731631794029 2024-11-15T00:49:55,510 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#C#compaction#264 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:55,510 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a7056081830e49a684f448f0313bafbf is 50, key is test_row_0/C:col10/1731631794031/Put/seqid=0 2024-11-15T00:49:55,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742141_1317 (size=12207) 2024-11-15T00:49:55,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:55,611 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:55,612 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T00:49:55,618 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ec78759e219f42c991d6cb96ea534abc_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631795610/Put/seqid=0 2024-11-15T00:49:55,620 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742142_1318 (size=14594) 2024-11-15T00:49:55,622 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:55,625 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ec78759e219f42c991d6cb96ea534abc_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ec78759e219f42c991d6cb96ea534abc_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:55,626 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e0944808dce945d1a1746bb1070d85d8, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:55,626 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e0944808dce945d1a1746bb1070d85d8 is 175, key is test_row_0/A:col10/1731631795610/Put/seqid=0 2024-11-15T00:49:55,629 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742143_1319 (size=39549) 2024-11-15T00:49:55,632 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e0944808dce945d1a1746bb1070d85d8 2024-11-15T00:49:55,639 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/2da5f7c528784928aea7e18d441ea6e0 is 50, key is test_row_0/B:col10/1731631795610/Put/seqid=0 2024-11-15T00:49:55,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742144_1320 (size=12001) 2024-11-15T00:49:55,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631855637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631855641, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,650 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631855646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,650 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,657 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631855648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631855647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,657 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,658 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,758 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631855756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,758 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,759 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631855766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,773 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,773 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631855766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,774 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,842 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/a7811d8e350f405cafc59fac5fbf3c28 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a7811d8e350f405cafc59fac5fbf3c28 2024-11-15T00:49:55,846 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/B of d7dd7a757dbb0ea573f9e0b990dd7e24 into a7811d8e350f405cafc59fac5fbf3c28(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:55,846 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:55,846 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/B, priority=13, startTime=1731631795399; duration=0sec 2024-11-15T00:49:55,846 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:55,846 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:B 2024-11-15T00:49:55,934 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a7056081830e49a684f448f0313bafbf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a7056081830e49a684f448f0313bafbf 2024-11-15T00:49:55,942 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/C of d7dd7a757dbb0ea573f9e0b990dd7e24 into a7056081830e49a684f448f0313bafbf(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:55,942 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:55,942 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/C, priority=13, startTime=1731631795400; duration=0sec 2024-11-15T00:49:55,942 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:55,942 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:C 2024-11-15T00:49:55,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631855967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,978 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631855976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:55,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:55,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631855979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,992 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:55,993 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,044 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/2da5f7c528784928aea7e18d441ea6e0 2024-11-15T00:49:56,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/5580aa2e3b7249efbeb344bfcabbca0a is 50, key is test_row_0/C:col10/1731631795610/Put/seqid=0 2024-11-15T00:49:56,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=97 2024-11-15T00:49:56,079 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:56,079 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:56,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-15T00:49:56,089 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:56,090 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:56,090 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:56,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:49:56,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742145_1321 (size=12001) 2024-11-15T00:49:56,105 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/5580aa2e3b7249efbeb344bfcabbca0a 2024-11-15T00:49:56,110 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e0944808dce945d1a1746bb1070d85d8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e0944808dce945d1a1746bb1070d85d8 2024-11-15T00:49:56,113 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e0944808dce945d1a1746bb1070d85d8, entries=200, sequenceid=118, filesize=38.6 K 2024-11-15T00:49:56,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/2da5f7c528784928aea7e18d441ea6e0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/2da5f7c528784928aea7e18d441ea6e0 2024-11-15T00:49:56,118 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/2da5f7c528784928aea7e18d441ea6e0, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T00:49:56,119 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/5580aa2e3b7249efbeb344bfcabbca0a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/5580aa2e3b7249efbeb344bfcabbca0a 2024-11-15T00:49:56,125 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/5580aa2e3b7249efbeb344bfcabbca0a, entries=150, sequenceid=118, filesize=11.7 K 2024-11-15T00:49:56,126 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 515ms, sequenceid=118, compaction requested=false 2024-11-15T00:49:56,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:49:56,243 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-15T00:49:56,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:56,243 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T00:49:56,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111581bea5a7797045cebe9b2574e9ea2c07_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631795629/Put/seqid=0 2024-11-15T00:49:56,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742146_1322 (size=12204) 2024-11-15T00:49:56,258 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:56,263 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111581bea5a7797045cebe9b2574e9ea2c07_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111581bea5a7797045cebe9b2574e9ea2c07_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:56,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/d81e922fc25a48d2b57385497762e947, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:56,265 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/d81e922fc25a48d2b57385497762e947 is 175, key is test_row_0/A:col10/1731631795629/Put/seqid=0 2024-11-15T00:49:56,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742147_1323 (size=31005) 2024-11-15T00:49:56,276 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=132, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/d81e922fc25a48d2b57385497762e947 2024-11-15T00:49:56,291 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:56,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/05b9974696364660a9382aa69e4e49cc is 50, key is test_row_0/B:col10/1731631795629/Put/seqid=0 2024-11-15T00:49:56,317 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742148_1324 (size=12051) 2024-11-15T00:49:56,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:49:56,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631856438, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,482 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631856456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,483 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,483 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631856457, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,484 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,573 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631856567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631856587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631856587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,598 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631856657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,669 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631856657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,669 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2230 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,669 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2221 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,670 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:49:56,724 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/05b9974696364660a9382aa69e4e49cc 2024-11-15T00:49:56,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/17ec2a33a6884e84a0ea8d6eaa0d235e is 50, key is test_row_0/C:col10/1731631795629/Put/seqid=0 2024-11-15T00:49:56,782 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631856777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,783 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,824 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631856807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,826 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,827 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:56,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631856807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:56,828 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:56,832 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742149_1325 (size=12051) 2024-11-15T00:49:57,093 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:57,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631857091, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:57,147 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631857141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,148 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631857141, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:57,148 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,149 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:57,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:49:57,234 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=132 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/17ec2a33a6884e84a0ea8d6eaa0d235e 2024-11-15T00:49:57,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/d81e922fc25a48d2b57385497762e947 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/d81e922fc25a48d2b57385497762e947 2024-11-15T00:49:57,247 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/d81e922fc25a48d2b57385497762e947, entries=150, sequenceid=132, filesize=30.3 K 2024-11-15T00:49:57,252 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/05b9974696364660a9382aa69e4e49cc as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/05b9974696364660a9382aa69e4e49cc 2024-11-15T00:49:57,260 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/05b9974696364660a9382aa69e4e49cc, entries=150, sequenceid=132, filesize=11.8 K 2024-11-15T00:49:57,264 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/17ec2a33a6884e84a0ea8d6eaa0d235e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17ec2a33a6884e84a0ea8d6eaa0d235e 2024-11-15T00:49:57,280 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17ec2a33a6884e84a0ea8d6eaa0d235e, entries=150, sequenceid=132, filesize=11.8 K 2024-11-15T00:49:57,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1041ms, sequenceid=132, compaction requested=true 2024-11-15T00:49:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:57,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-15T00:49:57,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-15T00:49:57,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-15T00:49:57,291 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.1990 sec 2024-11-15T00:49:57,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 1.2130 sec 2024-11-15T00:49:57,614 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:57,614 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T00:49:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:57,631 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153a669c0c554949ad99c86b3cad42144d_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631796434/Put/seqid=0 2024-11-15T00:49:57,637 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742150_1326 (size=14794) 2024-11-15T00:49:57,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631857658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:57,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:57,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631857658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,668 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:57,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,669 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:57,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:57,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631857667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,678 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:57,796 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:57,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631857787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:57,796 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:58,016 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:58,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631858007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,016 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,017 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:58,038 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:58,049 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153a669c0c554949ad99c86b3cad42144d_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a669c0c554949ad99c86b3cad42144d_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:58,049 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/7444c1de8a424f28a2f632b1d53b7601, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:58,050 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/7444c1de8a424f28a2f632b1d53b7601 is 175, key is test_row_0/A:col10/1731631796434/Put/seqid=0 2024-11-15T00:49:58,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742151_1327 (size=39749) 2024-11-15T00:49:58,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T00:49:58,196 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T00:49:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-15T00:49:58,239 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:49:58,239 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:49:58,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-15T00:49:58,242 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:49:58,243 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:49:58,243 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:49:58,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:49:58,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:58,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631858328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:58,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:49:58,396 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:49:58,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,397 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,397 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,476 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=158, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/7444c1de8a424f28a2f632b1d53b7601 2024-11-15T00:49:58,510 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/6005653e88ca4151912589fa68030303 is 50, key is test_row_0/B:col10/1731631796434/Put/seqid=0 2024-11-15T00:49:58,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742152_1328 (size=12151) 2024-11-15T00:49:58,545 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/6005653e88ca4151912589fa68030303 2024-11-15T00:49:58,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:49:58,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:58,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:49:58,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,567 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/311d01c593b6426980a8fc1acc2285ba is 50, key is test_row_0/C:col10/1731631796434/Put/seqid=0 2024-11-15T00:49:58,611 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742153_1329 (size=12151) 2024-11-15T00:49:58,613 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=158 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/311d01c593b6426980a8fc1acc2285ba 2024-11-15T00:49:58,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/7444c1de8a424f28a2f632b1d53b7601 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7444c1de8a424f28a2f632b1d53b7601 2024-11-15T00:49:58,659 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7444c1de8a424f28a2f632b1d53b7601, entries=200, sequenceid=158, filesize=38.8 K 2024-11-15T00:49:58,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/6005653e88ca4151912589fa68030303 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6005653e88ca4151912589fa68030303 2024-11-15T00:49:58,667 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6005653e88ca4151912589fa68030303, entries=150, sequenceid=158, filesize=11.9 K 2024-11-15T00:49:58,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/311d01c593b6426980a8fc1acc2285ba as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/311d01c593b6426980a8fc1acc2285ba 2024-11-15T00:49:58,690 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631858677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:58,691 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:58,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631858677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:58,691 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4252 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:58,692 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2234 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,692 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:58,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:58,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631858687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:58,696 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2240 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:49:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,696 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:58,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/311d01c593b6426980a8fc1acc2285ba, entries=150, sequenceid=158, filesize=11.9 K 2024-11-15T00:49:58,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1086ms, sequenceid=158, compaction requested=true 2024-11-15T00:49:58,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:58,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:49:58,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:58,700 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:58,700 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:58,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:49:58,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:58,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:49:58,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:58,701 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141464 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:58,701 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:58,701 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/A is initiating minor compaction (all files) 2024-11-15T00:49:58,701 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/B is initiating minor compaction (all files) 2024-11-15T00:49:58,701 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/B in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,701 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/A in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,701 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a7811d8e350f405cafc59fac5fbf3c28, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/2da5f7c528784928aea7e18d441ea6e0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/05b9974696364660a9382aa69e4e49cc, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6005653e88ca4151912589fa68030303] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=47.3 K 2024-11-15T00:49:58,701 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/295a26ed27a748cbbd872036a6103740, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e0944808dce945d1a1746bb1070d85d8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/d81e922fc25a48d2b57385497762e947, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7444c1de8a424f28a2f632b1d53b7601] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=138.1 K 2024-11-15T00:49:58,701 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,701 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/295a26ed27a748cbbd872036a6103740, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e0944808dce945d1a1746bb1070d85d8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/d81e922fc25a48d2b57385497762e947, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7444c1de8a424f28a2f632b1d53b7601] 2024-11-15T00:49:58,702 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a7811d8e350f405cafc59fac5fbf3c28, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731631794029 2024-11-15T00:49:58,702 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 295a26ed27a748cbbd872036a6103740, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731631794029 2024-11-15T00:49:58,702 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2da5f7c528784928aea7e18d441ea6e0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731631794436 2024-11-15T00:49:58,702 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0944808dce945d1a1746bb1070d85d8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731631794436 2024-11-15T00:49:58,702 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 05b9974696364660a9382aa69e4e49cc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731631795623 2024-11-15T00:49:58,702 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d81e922fc25a48d2b57385497762e947, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731631795623 2024-11-15T00:49:58,702 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 6005653e88ca4151912589fa68030303, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731631796434 2024-11-15T00:49:58,702 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7444c1de8a424f28a2f632b1d53b7601, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731631796434 2024-11-15T00:49:58,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:58,710 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:49:58,710 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:49:58,719 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#B#compaction#274 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:58,719 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/34bf5399c0ae4a199e8341035e9ae564 is 50, key is test_row_0/B:col10/1731631796434/Put/seqid=0 2024-11-15T00:49:58,720 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:49:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,730 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:58,759 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ec76af5f357947fc978e9186306b921f_d7dd7a757dbb0ea573f9e0b990dd7e24 store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:58,761 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ec76af5f357947fc978e9186306b921f_d7dd7a757dbb0ea573f9e0b990dd7e24, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:58,761 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ec76af5f357947fc978e9186306b921f_d7dd7a757dbb0ea573f9e0b990dd7e24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:58,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c5b81b11f4d449738a4f5e022f4c779a_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631797651/Put/seqid=0 2024-11-15T00:49:58,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742154_1330 (size=12493) 2024-11-15T00:49:58,819 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/34bf5399c0ae4a199e8341035e9ae564 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/34bf5399c0ae4a199e8341035e9ae564 2024-11-15T00:49:58,823 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/B of d7dd7a757dbb0ea573f9e0b990dd7e24 into 34bf5399c0ae4a199e8341035e9ae564(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:58,823 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:58,823 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/B, priority=12, startTime=1731631798700; duration=0sec 2024-11-15T00:49:58,824 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:49:58,824 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:B 2024-11-15T00:49:58,824 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:49:58,825 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48410 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:49:58,825 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/C is initiating minor compaction (all files) 2024-11-15T00:49:58,825 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/C in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,825 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a7056081830e49a684f448f0313bafbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/5580aa2e3b7249efbeb344bfcabbca0a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17ec2a33a6884e84a0ea8d6eaa0d235e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/311d01c593b6426980a8fc1acc2285ba] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=47.3 K 2024-11-15T00:49:58,825 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a7056081830e49a684f448f0313bafbf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=93, earliestPutTs=1731631794029 2024-11-15T00:49:58,826 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5580aa2e3b7249efbeb344bfcabbca0a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1731631794436 2024-11-15T00:49:58,826 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 17ec2a33a6884e84a0ea8d6eaa0d235e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=132, earliestPutTs=1731631795623 2024-11-15T00:49:58,826 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 311d01c593b6426980a8fc1acc2285ba, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731631796434 2024-11-15T00:49:58,838 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#C#compaction#277 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:58,838 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/36c0b0ca18274ada9cd4def07ca7b43b is 50, key is test_row_0/C:col10/1731631796434/Put/seqid=0 2024-11-15T00:49:58,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742156_1332 (size=14794) 2024-11-15T00:49:58,840 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:58,846 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742155_1331 (size=4469) 2024-11-15T00:49:58,848 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c5b81b11f4d449738a4f5e022f4c779a_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c5b81b11f4d449738a4f5e022f4c779a_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:58,849 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742157_1333 (size=12493) 2024-11-15T00:49:58,856 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/c75d62ce6d9b4646b378888e0eac48ea, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:58,856 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/c75d62ce6d9b4646b378888e0eac48ea is 175, key is test_row_0/A:col10/1731631797651/Put/seqid=0 2024-11-15T00:49:58,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742158_1334 (size=39749) 2024-11-15T00:49:58,863 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=169, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/c75d62ce6d9b4646b378888e0eac48ea 2024-11-15T00:49:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:49:58,873 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:49:58,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:58,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:58,874 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:49:58,878 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/a05a66600e684669841ddf5ba1dd3934 is 50, key is test_row_0/B:col10/1731631797651/Put/seqid=0 2024-11-15T00:49:58,887 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742159_1335 (size=12151) 2024-11-15T00:49:58,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/a05a66600e684669841ddf5ba1dd3934 2024-11-15T00:49:58,904 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/cbfa82c85535432da6f9ad90cc66f5d2 is 50, key is test_row_0/C:col10/1731631797651/Put/seqid=0 2024-11-15T00:49:58,923 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:58,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631858910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:58,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742160_1336 (size=12151) 2024-11-15T00:49:58,932 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/cbfa82c85535432da6f9ad90cc66f5d2 2024-11-15T00:49:58,945 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/c75d62ce6d9b4646b378888e0eac48ea as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c75d62ce6d9b4646b378888e0eac48ea 2024-11-15T00:49:58,948 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:58,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631858924, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:58,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c75d62ce6d9b4646b378888e0eac48ea, entries=200, sequenceid=169, filesize=38.8 K 2024-11-15T00:49:58,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:58,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:58,958 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/a05a66600e684669841ddf5ba1dd3934 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a05a66600e684669841ddf5ba1dd3934 2024-11-15T00:49:58,963 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a05a66600e684669841ddf5ba1dd3934, entries=150, sequenceid=169, filesize=11.9 K 2024-11-15T00:49:58,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/cbfa82c85535432da6f9ad90cc66f5d2 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cbfa82c85535432da6f9ad90cc66f5d2 2024-11-15T00:49:58,968 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cbfa82c85535432da6f9ad90cc66f5d2, entries=150, sequenceid=169, filesize=11.9 K 2024-11-15T00:49:58,969 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 259ms, sequenceid=169, compaction requested=false 2024-11-15T00:49:58,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:59,028 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-15T00:49:59,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:59,028 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:49:59,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:59,035 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:49:59,035 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159ca878cf534a4fed87b7a54951e1c016_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631798907/Put/seqid=0 2024-11-15T00:49:59,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742161_1337 (size=12304) 2024-11-15T00:49:59,062 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:49:59,067 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159ca878cf534a4fed87b7a54951e1c016_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159ca878cf534a4fed87b7a54951e1c016_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:49:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e42536a1cada4779a0b9c70207e6ccd0, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:49:59,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e42536a1cada4779a0b9c70207e6ccd0 is 175, key is test_row_0/A:col10/1731631798907/Put/seqid=0 2024-11-15T00:49:59,092 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:59,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631859075, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:59,093 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:59,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631859076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,094 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:59,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742162_1338 (size=31105) 2024-11-15T00:49:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:59,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631859197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:59,212 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:59,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631859197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,214 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:59,215 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,216 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:59,248 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#A#compaction#275 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:49:59,249 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/bb19f02b773f4667aa16c3de90d8eec5 is 175, key is test_row_0/A:col10/1731631796434/Put/seqid=0 2024-11-15T00:49:59,267 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/36c0b0ca18274ada9cd4def07ca7b43b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/36c0b0ca18274ada9cd4def07ca7b43b 2024-11-15T00:49:59,282 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742163_1339 (size=31447) 2024-11-15T00:49:59,292 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/bb19f02b773f4667aa16c3de90d8eec5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/bb19f02b773f4667aa16c3de90d8eec5 2024-11-15T00:49:59,297 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/C of d7dd7a757dbb0ea573f9e0b990dd7e24 into 36c0b0ca18274ada9cd4def07ca7b43b(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:59,297 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:59,297 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/C, priority=12, startTime=1731631798700; duration=0sec 2024-11-15T00:49:59,297 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:59,297 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:C 2024-11-15T00:49:59,301 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/A of d7dd7a757dbb0ea573f9e0b990dd7e24 into bb19f02b773f4667aa16c3de90d8eec5(size=30.7 K), total size for store is 69.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:49:59,301 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:59,301 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/A, priority=12, startTime=1731631798700; duration=0sec 2024-11-15T00:49:59,301 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:49:59,301 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:A 2024-11-15T00:49:59,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:49:59,442 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:59,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631859417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:59,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,444 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:59,466 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:59,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631859427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,468 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:59,516 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e42536a1cada4779a0b9c70207e6ccd0 2024-11-15T00:49:59,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/41a22cfe3b544130912fa3d794d85721 is 50, key is test_row_0/B:col10/1731631798907/Put/seqid=0 2024-11-15T00:49:59,753 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742164_1340 (size=12151) 2024-11-15T00:49:59,754 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/41a22cfe3b544130912fa3d794d85721 2024-11-15T00:49:59,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/20211d1c37d541db8edda0d3475f1b82 is 50, key is test_row_0/C:col10/1731631798907/Put/seqid=0 2024-11-15T00:49:59,785 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:59,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631859753, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:59,791 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:49:59,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631859777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:49:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:49:59,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:49:59,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742165_1341 (size=12151) 2024-11-15T00:49:59,818 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/20211d1c37d541db8edda0d3475f1b82 2024-11-15T00:49:59,836 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e42536a1cada4779a0b9c70207e6ccd0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e42536a1cada4779a0b9c70207e6ccd0 2024-11-15T00:49:59,843 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e42536a1cada4779a0b9c70207e6ccd0, entries=150, sequenceid=195, filesize=30.4 K 2024-11-15T00:49:59,852 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/41a22cfe3b544130912fa3d794d85721 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/41a22cfe3b544130912fa3d794d85721 2024-11-15T00:49:59,868 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/41a22cfe3b544130912fa3d794d85721, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T00:49:59,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/20211d1c37d541db8edda0d3475f1b82 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/20211d1c37d541db8edda0d3475f1b82 2024-11-15T00:49:59,889 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/20211d1c37d541db8edda0d3475f1b82, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T00:49:59,890 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 862ms, sequenceid=195, compaction requested=true 2024-11-15T00:49:59,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:49:59,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:49:59,890 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-15T00:49:59,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-15T00:49:59,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-15T00:49:59,894 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6490 sec 2024-11-15T00:49:59,897 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 1.6560 sec 2024-11-15T00:50:00,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:00,302 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:00,302 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T00:50:00,314 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155deff2be8eb44af18dc9c0dc70d027db_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631799068/Put/seqid=0 2024-11-15T00:50:00,318 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742166_1342 (size=14794) 2024-11-15T00:50:00,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-15T00:50:00,396 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:00,397 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-15T00:50:00,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:50:00,398 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:00,399 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:00,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:00,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631860393, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:00,408 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:00,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631860394, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,409 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:00,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:50:00,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631860517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:00,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631860517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,525 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:00,552 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T00:50:00,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:00,553 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:00,553 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:00,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:00,704 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:00,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631860696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:00,705 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,706 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:00,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:00,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631860708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:00,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T00:50:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:00,709 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:00,709 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4253 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:00,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:00,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,709 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:00,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:00,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:50:00,722 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:00,729 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411155deff2be8eb44af18dc9c0dc70d027db_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155deff2be8eb44af18dc9c0dc70d027db_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:00,730 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/96b57c9041f54640bfe1517b21586710, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:00,730 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/96b57c9041f54640bfe1517b21586710 is 175, key is test_row_0/A:col10/1731631799068/Put/seqid=0 2024-11-15T00:50:00,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:00,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631860741, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:00,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:00,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631860743, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:00,762 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:00,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742167_1343 (size=39749) 2024-11-15T00:50:00,772 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=210, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/96b57c9041f54640bfe1517b21586710 2024-11-15T00:50:00,799 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/e75f262cda0d4d598703d09c7dc2fb72 is 50, key is test_row_0/B:col10/1731631799068/Put/seqid=0 2024-11-15T00:50:00,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742168_1344 (size=12151) 2024-11-15T00:50:00,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/e75f262cda0d4d598703d09c7dc2fb72 2024-11-15T00:50:00,858 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a0eb80d4c39b407eac571afc6110458a is 50, key is test_row_0/C:col10/1731631799068/Put/seqid=0 2024-11-15T00:50:00,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T00:50:00,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:00,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:00,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:00,871 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:00,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:00,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:00,886 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742169_1345 (size=12151) 2024-11-15T00:50:00,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=210 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a0eb80d4c39b407eac571afc6110458a 2024-11-15T00:50:00,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/96b57c9041f54640bfe1517b21586710 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/96b57c9041f54640bfe1517b21586710 2024-11-15T00:50:00,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/96b57c9041f54640bfe1517b21586710, entries=200, sequenceid=210, filesize=38.8 K 2024-11-15T00:50:00,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/e75f262cda0d4d598703d09c7dc2fb72 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/e75f262cda0d4d598703d09c7dc2fb72 2024-11-15T00:50:00,949 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/e75f262cda0d4d598703d09c7dc2fb72, entries=150, sequenceid=210, filesize=11.9 K 2024-11-15T00:50:00,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a0eb80d4c39b407eac571afc6110458a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a0eb80d4c39b407eac571afc6110458a 2024-11-15T00:50:00,956 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a0eb80d4c39b407eac571afc6110458a, entries=150, sequenceid=210, filesize=11.9 K 2024-11-15T00:50:00,957 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 655ms, sequenceid=210, compaction requested=true 2024-11-15T00:50:00,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:00,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:00,957 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:00,957 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:00,957 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:00,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:00,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:00,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:00,958 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:00,959 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142050 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:00,959 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/A is initiating minor compaction (all files) 2024-11-15T00:50:00,959 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/A in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:00,959 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/bb19f02b773f4667aa16c3de90d8eec5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c75d62ce6d9b4646b378888e0eac48ea, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e42536a1cada4779a0b9c70207e6ccd0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/96b57c9041f54640bfe1517b21586710] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=138.7 K 2024-11-15T00:50:00,959 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:00,959 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/bb19f02b773f4667aa16c3de90d8eec5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c75d62ce6d9b4646b378888e0eac48ea, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e42536a1cada4779a0b9c70207e6ccd0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/96b57c9041f54640bfe1517b21586710] 2024-11-15T00:50:00,959 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:00,959 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/B is initiating minor compaction (all files) 2024-11-15T00:50:00,960 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/B in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:00,960 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/34bf5399c0ae4a199e8341035e9ae564, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a05a66600e684669841ddf5ba1dd3934, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/41a22cfe3b544130912fa3d794d85721, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/e75f262cda0d4d598703d09c7dc2fb72] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=47.8 K 2024-11-15T00:50:00,960 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb19f02b773f4667aa16c3de90d8eec5, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731631796434 2024-11-15T00:50:00,961 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 34bf5399c0ae4a199e8341035e9ae564, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731631796434 2024-11-15T00:50:00,961 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c75d62ce6d9b4646b378888e0eac48ea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731631797647 2024-11-15T00:50:00,961 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a05a66600e684669841ddf5ba1dd3934, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731631797647 2024-11-15T00:50:00,961 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 41a22cfe3b544130912fa3d794d85721, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731631798876 2024-11-15T00:50:00,961 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e42536a1cada4779a0b9c70207e6ccd0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731631798876 2024-11-15T00:50:00,961 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e75f262cda0d4d598703d09c7dc2fb72, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731631799068 2024-11-15T00:50:00,962 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96b57c9041f54640bfe1517b21586710, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731631799068 2024-11-15T00:50:00,980 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#B#compaction#286 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:00,981 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/bffd343afc63492ba438d783567808f2 is 50, key is test_row_0/B:col10/1731631799068/Put/seqid=0 2024-11-15T00:50:00,983 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:00,997 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411152d4550d3fa3142719b2dbbd8f716ac2d_d7dd7a757dbb0ea573f9e0b990dd7e24 store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:00,999 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411152d4550d3fa3142719b2dbbd8f716ac2d_d7dd7a757dbb0ea573f9e0b990dd7e24, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:00,999 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152d4550d3fa3142719b2dbbd8f716ac2d_d7dd7a757dbb0ea573f9e0b990dd7e24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:01,001 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742170_1346 (size=12629) 2024-11-15T00:50:01,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742171_1347 (size=4469) 2024-11-15T00:50:01,015 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#A#compaction#287 average throughput is 0.79 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:01,015 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/81dba6b21bc9460590e4f2da9a889973 is 175, key is test_row_0/A:col10/1731631799068/Put/seqid=0 2024-11-15T00:50:01,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-15T00:50:01,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:01,024 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T00:50:01,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:50:01,063 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:01,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:01,066 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742172_1348 (size=31583) 2024-11-15T00:50:01,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115be357ce96b6f432cae47b23f86d5dacd_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631800393/Put/seqid=0 2024-11-15T00:50:01,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742173_1349 (size=12304) 2024-11-15T00:50:01,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:01,087 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115be357ce96b6f432cae47b23f86d5dacd_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115be357ce96b6f432cae47b23f86d5dacd_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:01,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/ce1df1de587f4745b438e3d2b459dec4, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:01,088 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/ce1df1de587f4745b438e3d2b459dec4 is 175, key is test_row_0/A:col10/1731631800393/Put/seqid=0 2024-11-15T00:50:01,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742174_1350 (size=31105) 2024-11-15T00:50:01,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631861167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:01,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:01,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631861168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,184 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,186 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:01,300 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:01,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631861288, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,300 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:01,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:01,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631861300, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,315 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:01,426 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/bffd343afc63492ba438d783567808f2 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/bffd343afc63492ba438d783567808f2 2024-11-15T00:50:01,431 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/B of d7dd7a757dbb0ea573f9e0b990dd7e24 into bffd343afc63492ba438d783567808f2(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:01,431 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:01,431 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/B, priority=12, startTime=1731631800957; duration=0sec 2024-11-15T00:50:01,431 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:01,432 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:B 2024-11-15T00:50:01,432 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:01,437 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48946 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:01,438 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/C is initiating minor compaction (all files) 2024-11-15T00:50:01,438 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/C in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:01,438 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/36c0b0ca18274ada9cd4def07ca7b43b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cbfa82c85535432da6f9ad90cc66f5d2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/20211d1c37d541db8edda0d3475f1b82, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a0eb80d4c39b407eac571afc6110458a] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=47.8 K 2024-11-15T00:50:01,441 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 36c0b0ca18274ada9cd4def07ca7b43b, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=158, earliestPutTs=1731631796434 2024-11-15T00:50:01,449 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting cbfa82c85535432da6f9ad90cc66f5d2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1731631797647 2024-11-15T00:50:01,457 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 20211d1c37d541db8edda0d3475f1b82, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731631798876 2024-11-15T00:50:01,457 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a0eb80d4c39b407eac571afc6110458a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731631799068 2024-11-15T00:50:01,482 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/81dba6b21bc9460590e4f2da9a889973 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/81dba6b21bc9460590e4f2da9a889973 2024-11-15T00:50:01,494 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/A of d7dd7a757dbb0ea573f9e0b990dd7e24 into 81dba6b21bc9460590e4f2da9a889973(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:01,495 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:01,495 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/A, priority=12, startTime=1731631800957; duration=0sec 2024-11-15T00:50:01,495 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:01,495 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:A 2024-11-15T00:50:01,500 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#C#compaction#289 average throughput is 1.09 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:01,505 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/cab7708decb148ec970d2df82d0a228f is 50, key is test_row_0/C:col10/1731631799068/Put/seqid=0 2024-11-15T00:50:01,515 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:01,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631861508, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,516 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:01,531 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:01,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631861517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:01,531 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,532 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:01,533 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=233, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/ce1df1de587f4745b438e3d2b459dec4 2024-11-15T00:50:01,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:50:01,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742175_1351 (size=12629) 2024-11-15T00:50:01,568 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/ee91687e50cc42ac9f211dfc4e620309 is 50, key is test_row_0/B:col10/1731631800393/Put/seqid=0 2024-11-15T00:50:01,581 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/cab7708decb148ec970d2df82d0a228f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cab7708decb148ec970d2df82d0a228f 2024-11-15T00:50:01,591 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/C of d7dd7a757dbb0ea573f9e0b990dd7e24 into cab7708decb148ec970d2df82d0a228f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:01,591 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:01,591 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/C, priority=12, startTime=1731631800958; duration=0sec 2024-11-15T00:50:01,591 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:01,591 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:C 2024-11-15T00:50:01,602 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742176_1352 (size=12151) 2024-11-15T00:50:01,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:01,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631861827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:01,836 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,837 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:01,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:01,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631861837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:01,853 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:02,003 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/ee91687e50cc42ac9f211dfc4e620309 2024-11-15T00:50:02,009 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/61043d7ce37d4f2580057b57ff9549b3 is 50, key is test_row_0/C:col10/1731631800393/Put/seqid=0 2024-11-15T00:50:02,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742177_1353 (size=12151) 2024-11-15T00:50:02,015 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=233 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/61043d7ce37d4f2580057b57ff9549b3 2024-11-15T00:50:02,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/ce1df1de587f4745b438e3d2b459dec4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ce1df1de587f4745b438e3d2b459dec4 2024-11-15T00:50:02,031 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ce1df1de587f4745b438e3d2b459dec4, entries=150, sequenceid=233, filesize=30.4 K 2024-11-15T00:50:02,032 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/ee91687e50cc42ac9f211dfc4e620309 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ee91687e50cc42ac9f211dfc4e620309 2024-11-15T00:50:02,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,037 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ee91687e50cc42ac9f211dfc4e620309, entries=150, sequenceid=233, filesize=11.9 K 2024-11-15T00:50:02,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/61043d7ce37d4f2580057b57ff9549b3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/61043d7ce37d4f2580057b57ff9549b3 2024-11-15T00:50:02,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,044 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/61043d7ce37d4f2580057b57ff9549b3, entries=150, sequenceid=233, filesize=11.9 K 2024-11-15T00:50:02,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,045 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1021ms, sequenceid=233, compaction requested=false 2024-11-15T00:50:02,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:02,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:02,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-15T00:50:02,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-15T00:50:02,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-15T00:50:02,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6480 sec 2024-11-15T00:50:02,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,051 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 1.6530 sec 2024-11-15T00:50:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,062 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,064 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,072 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,074 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,076 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,083 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,092 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,101 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,107 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,108 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,114 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,125 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,140 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,142 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,143 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,151 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,153 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,155 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:02,403 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:02,404 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-15T00:50:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153e32254f7de145be9c6671126b651749_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631802402/Put/seqid=0 2024-11-15T00:50:02,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742178_1354 (size=12304) 2024-11-15T00:50:02,438 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,445 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411153e32254f7de145be9c6671126b651749_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153e32254f7de145be9c6671126b651749_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:02,457 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f2d9ea92144b4e34a4b4b4ee3225f40c, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:02,458 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f2d9ea92144b4e34a4b4b4ee3225f40c is 175, key is test_row_0/A:col10/1731631802402/Put/seqid=0 2024-11-15T00:50:02,472 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742179_1355 (size=31101) 2024-11-15T00:50:02,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-15T00:50:02,546 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:02,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631862537, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:02,547 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,547 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:02,547 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:02,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-15T00:50:02,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T00:50:02,549 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:02,549 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:02,550 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:02,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:02,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631862547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:02,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T00:50:02,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:02,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631862657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:02,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:02,667 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631862657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,668 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:02,702 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T00:50:02,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:02,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:02,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:02,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:02,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:02,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:02,707 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:02,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38092 deadline: 1731631862707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:02,709 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8270 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,710 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:02,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T00:50:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:02,855 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:02,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:02,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:02,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T00:50:02,873 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=250, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f2d9ea92144b4e34a4b4b4ee3225f40c 2024-11-15T00:50:02,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:02,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631862877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,888 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:02,888 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:02,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631862877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:02,889 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:02,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/859d2b5688aa4d96b883711ad24da937 is 50, key is test_row_0/B:col10/1731631802402/Put/seqid=0 2024-11-15T00:50:02,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742180_1356 (size=9757) 2024-11-15T00:50:03,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T00:50:03,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:03,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,013 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:03,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:03,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:03,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T00:50:03,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:03,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,171 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:03,171 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:03,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:03,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T00:50:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631863196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:03,204 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:03,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631863196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:03,204 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:03,205 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:03,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T00:50:03,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:03,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:03,351 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/859d2b5688aa4d96b883711ad24da937 2024-11-15T00:50:03,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:03,374 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/17b3e344fc614bb8ba4c62f99066ad01 is 50, key is test_row_0/C:col10/1731631802402/Put/seqid=0 2024-11-15T00:50:03,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742181_1357 (size=9757) 2024-11-15T00:50:03,389 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=250 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/17b3e344fc614bb8ba4c62f99066ad01 2024-11-15T00:50:03,397 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f2d9ea92144b4e34a4b4b4ee3225f40c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f2d9ea92144b4e34a4b4b4ee3225f40c 2024-11-15T00:50:03,401 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f2d9ea92144b4e34a4b4b4ee3225f40c, entries=150, sequenceid=250, filesize=30.4 K 2024-11-15T00:50:03,402 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/859d2b5688aa4d96b883711ad24da937 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/859d2b5688aa4d96b883711ad24da937 2024-11-15T00:50:03,409 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/859d2b5688aa4d96b883711ad24da937, entries=100, sequenceid=250, filesize=9.5 K 2024-11-15T00:50:03,410 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/17b3e344fc614bb8ba4c62f99066ad01 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17b3e344fc614bb8ba4c62f99066ad01 2024-11-15T00:50:03,413 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17b3e344fc614bb8ba4c62f99066ad01, entries=100, sequenceid=250, filesize=9.5 K 2024-11-15T00:50:03,414 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1011ms, sequenceid=250, compaction requested=true 2024-11-15T00:50:03,414 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:03,414 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:03,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:03,414 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:03,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:03,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:03,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:03,415 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:50:03,415 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:03,416 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:03,416 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/A is initiating minor compaction (all files) 2024-11-15T00:50:03,416 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/A in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,416 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/81dba6b21bc9460590e4f2da9a889973, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ce1df1de587f4745b438e3d2b459dec4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f2d9ea92144b4e34a4b4b4ee3225f40c] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=91.6 K 2024-11-15T00:50:03,416 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,416 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/81dba6b21bc9460590e4f2da9a889973, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ce1df1de587f4745b438e3d2b459dec4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f2d9ea92144b4e34a4b4b4ee3225f40c] 2024-11-15T00:50:03,417 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 81dba6b21bc9460590e4f2da9a889973, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731631799068 2024-11-15T00:50:03,419 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:03,419 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ce1df1de587f4745b438e3d2b459dec4, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731631800364 2024-11-15T00:50:03,419 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/B is initiating minor compaction (all files) 2024-11-15T00:50:03,419 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/B in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,419 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/bffd343afc63492ba438d783567808f2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ee91687e50cc42ac9f211dfc4e620309, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/859d2b5688aa4d96b883711ad24da937] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=33.7 K 2024-11-15T00:50:03,419 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d9ea92144b4e34a4b4b4ee3225f40c, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731631801130 2024-11-15T00:50:03,419 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bffd343afc63492ba438d783567808f2, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731631799068 2024-11-15T00:50:03,419 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting ee91687e50cc42ac9f211dfc4e620309, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731631800364 2024-11-15T00:50:03,420 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 859d2b5688aa4d96b883711ad24da937, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731631801147 2024-11-15T00:50:03,429 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:03,430 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#B#compaction#295 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:03,430 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/0486aa721db44080a51f818e9188fcef is 50, key is test_row_0/B:col10/1731631802402/Put/seqid=0 2024-11-15T00:50:03,437 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115e981774927ca42e8a53145458889d9ba_d7dd7a757dbb0ea573f9e0b990dd7e24 store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:03,439 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115e981774927ca42e8a53145458889d9ba_d7dd7a757dbb0ea573f9e0b990dd7e24, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:03,439 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e981774927ca42e8a53145458889d9ba_d7dd7a757dbb0ea573f9e0b990dd7e24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:03,487 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742182_1358 (size=12731) 2024-11-15T00:50:03,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742183_1359 (size=4469) 2024-11-15T00:50:03,495 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#A#compaction#296 average throughput is 0.37 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:03,496 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/dd89145da4064eb58eccf587fa2b200c is 175, key is test_row_0/A:col10/1731631802402/Put/seqid=0 2024-11-15T00:50:03,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-15T00:50:03,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,505 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-15T00:50:03,513 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742184_1360 (size=31792) 2024-11-15T00:50:03,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e6b44f85fe4f4be5adb8f650099a5386_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631802533/Put/seqid=0 2024-11-15T00:50:03,538 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/dd89145da4064eb58eccf587fa2b200c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/dd89145da4064eb58eccf587fa2b200c 2024-11-15T00:50:03,546 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/A of d7dd7a757dbb0ea573f9e0b990dd7e24 into dd89145da4064eb58eccf587fa2b200c(size=31.0 K), total size for store is 31.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:03,546 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:03,546 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/A, priority=13, startTime=1731631803414; duration=0sec 2024-11-15T00:50:03,546 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:03,546 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:A 2024-11-15T00:50:03,546 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:03,550 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34537 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:03,550 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/C is initiating minor compaction (all files) 2024-11-15T00:50:03,567 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/C in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,567 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cab7708decb148ec970d2df82d0a228f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/61043d7ce37d4f2580057b57ff9549b3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17b3e344fc614bb8ba4c62f99066ad01] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=33.7 K 2024-11-15T00:50:03,570 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting cab7708decb148ec970d2df82d0a228f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=210, earliestPutTs=1731631799068 2024-11-15T00:50:03,570 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 61043d7ce37d4f2580057b57ff9549b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=233, earliestPutTs=1731631800364 2024-11-15T00:50:03,571 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 17b3e344fc614bb8ba4c62f99066ad01, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731631801147 2024-11-15T00:50:03,573 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742185_1361 (size=12454) 2024-11-15T00:50:03,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:03,582 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e6b44f85fe4f4be5adb8f650099a5386_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e6b44f85fe4f4be5adb8f650099a5386_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:03,584 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#C#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:03,584 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/7299f9113bcd4d2fb03067fb941f5311 is 50, key is test_row_0/C:col10/1731631802402/Put/seqid=0 2024-11-15T00:50:03,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/23e302648ea04b93a619348fd4975ffa, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:03,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/23e302648ea04b93a619348fd4975ffa is 175, key is test_row_0/A:col10/1731631802533/Put/seqid=0 2024-11-15T00:50:03,619 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742186_1362 (size=31255) 2024-11-15T00:50:03,628 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=272, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/23e302648ea04b93a619348fd4975ffa 2024-11-15T00:50:03,649 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742187_1363 (size=12731) 2024-11-15T00:50:03,657 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/7299f9113bcd4d2fb03067fb941f5311 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/7299f9113bcd4d2fb03067fb941f5311 2024-11-15T00:50:03,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/ac6c2042f8394bc28f0be3429ad67564 is 50, key is test_row_0/B:col10/1731631802533/Put/seqid=0 2024-11-15T00:50:03,663 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/C of d7dd7a757dbb0ea573f9e0b990dd7e24 into 7299f9113bcd4d2fb03067fb941f5311(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:03,663 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:03,663 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/C, priority=13, startTime=1731631803415; duration=0sec 2024-11-15T00:50:03,663 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:03,663 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:C 2024-11-15T00:50:03,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T00:50:03,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742188_1364 (size=12301) 2024-11-15T00:50:03,706 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/ac6c2042f8394bc28f0be3429ad67564 2024-11-15T00:50:03,715 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/79eea3377fa04d4787acb221447ce48b is 50, key is test_row_0/C:col10/1731631802533/Put/seqid=0 2024-11-15T00:50:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:03,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:03,726 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742189_1365 (size=12301) 2024-11-15T00:50:03,730 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=272 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/79eea3377fa04d4787acb221447ce48b 2024-11-15T00:50:03,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/23e302648ea04b93a619348fd4975ffa as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/23e302648ea04b93a619348fd4975ffa 2024-11-15T00:50:03,744 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/23e302648ea04b93a619348fd4975ffa, entries=150, sequenceid=272, filesize=30.5 K 2024-11-15T00:50:03,746 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/ac6c2042f8394bc28f0be3429ad67564 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ac6c2042f8394bc28f0be3429ad67564 2024-11-15T00:50:03,751 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ac6c2042f8394bc28f0be3429ad67564, entries=150, sequenceid=272, filesize=12.0 K 2024-11-15T00:50:03,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/79eea3377fa04d4787acb221447ce48b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/79eea3377fa04d4787acb221447ce48b 2024-11-15T00:50:03,755 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/79eea3377fa04d4787acb221447ce48b, entries=150, sequenceid=272, filesize=12.0 K 2024-11-15T00:50:03,756 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=46.96 KB/48090 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 251ms, sequenceid=272, compaction requested=false 2024-11-15T00:50:03,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:03,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:03,756 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-15T00:50:03,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-15T00:50:03,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-15T00:50:03,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2070 sec 2024-11-15T00:50:03,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:03,760 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:50:03,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:03,763 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 1.2140 sec 2024-11-15T00:50:03,774 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151f9e72353a9e418c83b6d878c0ec15bf_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631803745/Put/seqid=0 2024-11-15T00:50:03,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742190_1366 (size=14994) 2024-11-15T00:50:03,817 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:03,825 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411151f9e72353a9e418c83b6d878c0ec15bf_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151f9e72353a9e418c83b6d878c0ec15bf_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:03,826 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/2a5e6f564aaf4b64bce241da569964f2, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:03,827 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/2a5e6f564aaf4b64bce241da569964f2 is 175, key is test_row_0/A:col10/1731631803745/Put/seqid=0 2024-11-15T00:50:03,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742191_1367 (size=39949) 2024-11-15T00:50:03,900 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/0486aa721db44080a51f818e9188fcef as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/0486aa721db44080a51f818e9188fcef 2024-11-15T00:50:03,907 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/B of d7dd7a757dbb0ea573f9e0b990dd7e24 into 0486aa721db44080a51f818e9188fcef(size=12.4 K), total size for store is 24.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:03,907 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:03,907 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/B, priority=13, startTime=1731631803415; duration=0sec 2024-11-15T00:50:03,907 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:03,907 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:B 2024-11-15T00:50:03,992 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:03,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631863979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:03,993 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:03,993 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:03,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631863981, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:03,994 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:04,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631864101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:04,116 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:04,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631864101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,117 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:04,276 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/2a5e6f564aaf4b64bce241da569964f2 2024-11-15T00:50:04,293 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/13db97a681734e7dae9cbe69218b4af5 is 50, key is test_row_0/B:col10/1731631803745/Put/seqid=0 2024-11-15T00:50:04,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742192_1368 (size=12301) 2024-11-15T00:50:04,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631864328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:04,336 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:04,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631864328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:04,654 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631864647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,655 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:04,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631864647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:04,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-15T00:50:04,697 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:04,697 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:04,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-15T00:50:04,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T00:50:04,699 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:04,700 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:04,700 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:04,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/13db97a681734e7dae9cbe69218b4af5 2024-11-15T00:50:04,719 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a2bd99c801474d70b6af8ab2d1a22182 is 50, key is test_row_0/C:col10/1731631803745/Put/seqid=0 2024-11-15T00:50:04,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742193_1369 (size=12301) 2024-11-15T00:50:04,735 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:04,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38058 deadline: 1731631864727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:04,735 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8278 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,736 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:04,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:04,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38074 deadline: 1731631864739, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:04,747 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in d7dd7a757dbb0ea573f9e0b990dd7e24 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8291 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:04,747 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:04,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T00:50:04,853 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:04,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:04,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:04,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:04,853 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:04,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:04,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,008 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:05,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:05,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,008 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T00:50:05,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a2bd99c801474d70b6af8ab2d1a22182 2024-11-15T00:50:05,139 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/2a5e6f564aaf4b64bce241da569964f2 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2a5e6f564aaf4b64bce241da569964f2 2024-11-15T00:50:05,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2a5e6f564aaf4b64bce241da569964f2, entries=200, sequenceid=285, filesize=39.0 K 2024-11-15T00:50:05,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/13db97a681734e7dae9cbe69218b4af5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/13db97a681734e7dae9cbe69218b4af5 2024-11-15T00:50:05,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/13db97a681734e7dae9cbe69218b4af5, entries=150, sequenceid=285, filesize=12.0 K 2024-11-15T00:50:05,153 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a2bd99c801474d70b6af8ab2d1a22182 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2bd99c801474d70b6af8ab2d1a22182 2024-11-15T00:50:05,161 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,161 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,161 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,162 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2bd99c801474d70b6af8ab2d1a22182, entries=150, sequenceid=285, filesize=12.0 K 2024-11-15T00:50:05,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,164 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1405ms, sequenceid=285, compaction requested=true 2024-11-15T00:50:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:05,165 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:05,165 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:05,165 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:05,167 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:05,167 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/B is initiating minor compaction (all files) 2024-11-15T00:50:05,167 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/B in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,167 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/0486aa721db44080a51f818e9188fcef, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ac6c2042f8394bc28f0be3429ad67564, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/13db97a681734e7dae9cbe69218b4af5] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=36.5 K 2024-11-15T00:50:05,168 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102996 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:05,168 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/A is initiating minor compaction (all files) 2024-11-15T00:50:05,168 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/A in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,168 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/dd89145da4064eb58eccf587fa2b200c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/23e302648ea04b93a619348fd4975ffa, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2a5e6f564aaf4b64bce241da569964f2] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=100.6 K 2024-11-15T00:50:05,168 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,168 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/dd89145da4064eb58eccf587fa2b200c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/23e302648ea04b93a619348fd4975ffa, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2a5e6f564aaf4b64bce241da569964f2] 2024-11-15T00:50:05,169 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0486aa721db44080a51f818e9188fcef, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731631800364 2024-11-15T00:50:05,172 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ac6c2042f8394bc28f0be3429ad67564, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731631802531 2024-11-15T00:50:05,172 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd89145da4064eb58eccf587fa2b200c, keycount=150, bloomtype=ROW, size=31.0 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731631800364 2024-11-15T00:50:05,173 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23e302648ea04b93a619348fd4975ffa, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731631802531 2024-11-15T00:50:05,173 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 13db97a681734e7dae9cbe69218b4af5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731631803734 2024-11-15T00:50:05,176 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2a5e6f564aaf4b64bce241da569964f2, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731631803734 2024-11-15T00:50:05,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:05,180 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:05,180 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T00:50:05,185 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#B#compaction#304 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:05,185 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/eacfd2b0797247b397e7627345bb751b is 50, key is test_row_0/B:col10/1731631803745/Put/seqid=0 2024-11-15T00:50:05,186 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:05,205 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115c559c7e20b834c43b2cdd73ba7c23fca_d7dd7a757dbb0ea573f9e0b990dd7e24 store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:05,208 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115372136f5c4f748d595e17503c9f7cbc9_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631803945/Put/seqid=0 2024-11-15T00:50:05,208 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115c559c7e20b834c43b2cdd73ba7c23fca_d7dd7a757dbb0ea573f9e0b990dd7e24, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:05,209 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c559c7e20b834c43b2cdd73ba7c23fca_d7dd7a757dbb0ea573f9e0b990dd7e24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:05,210 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742194_1370 (size=12983) 2024-11-15T00:50:05,219 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/eacfd2b0797247b397e7627345bb751b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/eacfd2b0797247b397e7627345bb751b 2024-11-15T00:50:05,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742195_1371 (size=4469) 2024-11-15T00:50:05,223 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:05,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631865213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:05,228 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/B of d7dd7a757dbb0ea573f9e0b990dd7e24 into eacfd2b0797247b397e7627345bb751b(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:05,228 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:05,228 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/B, priority=13, startTime=1731631805165; duration=0sec 2024-11-15T00:50:05,228 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:05,228 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:B 2024-11-15T00:50:05,228 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:05,228 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:05,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631865223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:05,229 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742196_1372 (size=14994) 2024-11-15T00:50:05,231 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37333 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:05,231 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/C is initiating minor compaction (all files) 2024-11-15T00:50:05,231 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/C in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,231 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/7299f9113bcd4d2fb03067fb941f5311, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/79eea3377fa04d4787acb221447ce48b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2bd99c801474d70b6af8ab2d1a22182] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=36.5 K 2024-11-15T00:50:05,232 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7299f9113bcd4d2fb03067fb941f5311, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=250, earliestPutTs=1731631800364 2024-11-15T00:50:05,232 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 79eea3377fa04d4787acb221447ce48b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=272, earliestPutTs=1731631802531 2024-11-15T00:50:05,232 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a2bd99c801474d70b6af8ab2d1a22182, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731631803734 2024-11-15T00:50:05,252 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#C#compaction#307 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:05,253 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/3fe2715842624d75936aebd11b59d638 is 50, key is test_row_0/C:col10/1731631803745/Put/seqid=0 2024-11-15T00:50:05,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742197_1373 (size=12983) 2024-11-15T00:50:05,270 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/3fe2715842624d75936aebd11b59d638 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/3fe2715842624d75936aebd11b59d638 2024-11-15T00:50:05,283 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/C of d7dd7a757dbb0ea573f9e0b990dd7e24 into 3fe2715842624d75936aebd11b59d638(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:05,283 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:05,283 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/C, priority=13, startTime=1731631805165; duration=0sec 2024-11-15T00:50:05,283 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:05,283 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:C 2024-11-15T00:50:05,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:05,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,317 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,317 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T00:50:05,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:05,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631865333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,336 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:05,345 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:05,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631865337, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,345 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:05,470 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,471 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,471 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:05,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631865547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,554 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:05,567 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:05,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631865558, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:05,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,568 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:05,625 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#A#compaction#305 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:05,626 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/ceb15db3396b4f66a6fe9c3a18e52ee5 is 175, key is test_row_0/A:col10/1731631803745/Put/seqid=0 2024-11-15T00:50:05,630 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:05,632 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:05,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:05,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,632 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,635 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115372136f5c4f748d595e17503c9f7cbc9_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115372136f5c4f748d595e17503c9f7cbc9_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:05,636 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/c0fcab67774a4259a6203bf9062232a1, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:05,637 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/c0fcab67774a4259a6203bf9062232a1 is 175, key is test_row_0/A:col10/1731631803945/Put/seqid=0 2024-11-15T00:50:05,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742198_1374 (size=31937) 2024-11-15T00:50:05,664 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742199_1375 (size=39949) 2024-11-15T00:50:05,786 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:05,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:05,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,787 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T00:50:05,858 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:05,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631865857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,859 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:05,881 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:05,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631865877, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:05,882 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:05,942 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:05,943 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,943 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:05,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,055 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/ceb15db3396b4f66a6fe9c3a18e52ee5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ceb15db3396b4f66a6fe9c3a18e52ee5 2024-11-15T00:50:06,062 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/A of d7dd7a757dbb0ea573f9e0b990dd7e24 into ceb15db3396b4f66a6fe9c3a18e52ee5(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:06,062 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:06,062 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/A, priority=13, startTime=1731631805165; duration=0sec 2024-11-15T00:50:06,062 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:06,062 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:A 2024-11-15T00:50:06,064 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=313, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/c0fcab67774a4259a6203bf9062232a1 2024-11-15T00:50:06,072 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/d7378557849d4ed0b1cf01ef79542177 is 50, key is test_row_0/B:col10/1731631803945/Put/seqid=0 2024-11-15T00:50:06,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742200_1376 (size=12301) 2024-11-15T00:50:06,097 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:06,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:06,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:06,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:06,097 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:06,250 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,374 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:06,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631866367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:06,374 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:06,394 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:06,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631866386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:06,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:06,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:06,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:06,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:06,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:06,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,484 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/d7378557849d4ed0b1cf01ef79542177 2024-11-15T00:50:06,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/fce979540b29421d89daf8c637638859 is 50, key is test_row_0/C:col10/1731631803945/Put/seqid=0 2024-11-15T00:50:06,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742201_1377 (size=12301) 2024-11-15T00:50:06,547 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=313 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/fce979540b29421d89daf8c637638859 2024-11-15T00:50:06,560 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/c0fcab67774a4259a6203bf9062232a1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c0fcab67774a4259a6203bf9062232a1 2024-11-15T00:50:06,569 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c0fcab67774a4259a6203bf9062232a1, entries=200, sequenceid=313, filesize=39.0 K 2024-11-15T00:50:06,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/d7378557849d4ed0b1cf01ef79542177 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/d7378557849d4ed0b1cf01ef79542177 2024-11-15T00:50:06,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/d7378557849d4ed0b1cf01ef79542177, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T00:50:06,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/fce979540b29421d89daf8c637638859 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/fce979540b29421d89daf8c637638859 2024-11-15T00:50:06,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:06,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:06,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:06,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:06,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] handler.RSProcedureHandler(58): pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=108 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=108 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:06,586 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/fce979540b29421d89daf8c637638859, entries=150, sequenceid=313, filesize=12.0 K 2024-11-15T00:50:06,587 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-15T00:50:06,588 DEBUG [MemStoreFlusher.0 {}] regionserver.StoreScanner(1005): StoreScanner already closing. There is no need to updateReaders 2024-11-15T00:50:06,588 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1408ms, sequenceid=313, compaction requested=false 2024-11-15T00:50:06,588 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:06,739 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-15T00:50:06,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:06,739 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T00:50:06,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152bb3d36c60364f039b94378b06e50e1e_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631805221/Put/seqid=0 2024-11-15T00:50:06,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742202_1378 (size=12454) 2024-11-15T00:50:06,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:06,775 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152bb3d36c60364f039b94378b06e50e1e_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152bb3d36c60364f039b94378b06e50e1e_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/cc36814be8084bd49be4eb3a6d79becd, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:06,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/cc36814be8084bd49be4eb3a6d79becd is 175, key is test_row_0/A:col10/1731631805221/Put/seqid=0 2024-11-15T00:50:06,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742203_1379 (size=31255) 2024-11-15T00:50:06,801 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=325, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/cc36814be8084bd49be4eb3a6d79becd 2024-11-15T00:50:06,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/7d5c3dc51a8b4b8ca98c560bc51f3d22 is 50, key is test_row_0/B:col10/1731631805221/Put/seqid=0 2024-11-15T00:50:06,829 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742204_1380 (size=12301) 2024-11-15T00:50:06,832 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/7d5c3dc51a8b4b8ca98c560bc51f3d22 2024-11-15T00:50:06,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a344043eb80341d3b4b783582695b503 is 50, key is test_row_0/C:col10/1731631805221/Put/seqid=0 2024-11-15T00:50:06,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T00:50:06,883 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742205_1381 (size=12301) 2024-11-15T00:50:07,285 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a344043eb80341d3b4b783582695b503 2024-11-15T00:50:07,291 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/cc36814be8084bd49be4eb3a6d79becd as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/cc36814be8084bd49be4eb3a6d79becd 2024-11-15T00:50:07,295 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/cc36814be8084bd49be4eb3a6d79becd, entries=150, sequenceid=325, filesize=30.5 K 2024-11-15T00:50:07,297 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/7d5c3dc51a8b4b8ca98c560bc51f3d22 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7d5c3dc51a8b4b8ca98c560bc51f3d22 2024-11-15T00:50:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,301 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7d5c3dc51a8b4b8ca98c560bc51f3d22, entries=150, sequenceid=325, filesize=12.0 K 2024-11-15T00:50:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/a344043eb80341d3b4b783582695b503 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a344043eb80341d3b4b783582695b503 2024-11-15T00:50:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,310 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a344043eb80341d3b4b783582695b503, entries=150, sequenceid=325, filesize=12.0 K 2024-11-15T00:50:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,311 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 572ms, sequenceid=325, compaction requested=true 2024-11-15T00:50:07,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:07,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:07,311 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-15T00:50:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-15T00:50:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-15T00:50:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,314 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6120 sec 2024-11-15T00:50:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,316 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 2.6180 sec 2024-11-15T00:50:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:07,519 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:07,519 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:50:07,527 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159a2c3b1aac5543af9dca5348939020a7_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631807517/Put/seqid=0 2024-11-15T00:50:07,560 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742206_1382 (size=14994) 2024-11-15T00:50:07,722 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631867708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:07,723 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631867708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:07,830 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631867828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:07,831 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:07,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631867828, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:07,831 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:07,956 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:07,960 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411159a2c3b1aac5543af9dca5348939020a7_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159a2c3b1aac5543af9dca5348939020a7_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:07,961 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/749ec543a222447c917e154e566e6d4c, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:07,961 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/749ec543a222447c917e154e566e6d4c is 175, key is test_row_0/A:col10/1731631807517/Put/seqid=0 2024-11-15T00:50:07,970 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742207_1383 (size=39949) 2024-11-15T00:50:08,042 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:08,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631868036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:08,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631868037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:08,359 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:08,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631868348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:08,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631868348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,360 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:08,372 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=336, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/749ec543a222447c917e154e566e6d4c 2024-11-15T00:50:08,393 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/5c32adf0572d4a54a12f308967c80041 is 50, key is test_row_0/B:col10/1731631807517/Put/seqid=0 2024-11-15T00:50:08,413 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742208_1384 (size=12301) 2024-11-15T00:50:08,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/5c32adf0572d4a54a12f308967c80041 2024-11-15T00:50:08,426 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/6330105da3614ce58b4a666b3bd3a901 is 50, key is test_row_0/C:col10/1731631807517/Put/seqid=0 2024-11-15T00:50:08,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742209_1385 (size=12301) 2024-11-15T00:50:08,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=336 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/6330105da3614ce58b4a666b3bd3a901 2024-11-15T00:50:08,445 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/749ec543a222447c917e154e566e6d4c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/749ec543a222447c917e154e566e6d4c 2024-11-15T00:50:08,448 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/749ec543a222447c917e154e566e6d4c, entries=200, sequenceid=336, filesize=39.0 K 2024-11-15T00:50:08,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/5c32adf0572d4a54a12f308967c80041 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/5c32adf0572d4a54a12f308967c80041 2024-11-15T00:50:08,455 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/5c32adf0572d4a54a12f308967c80041, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T00:50:08,456 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/6330105da3614ce58b4a666b3bd3a901 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/6330105da3614ce58b4a666b3bd3a901 2024-11-15T00:50:08,464 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/6330105da3614ce58b4a666b3bd3a901, entries=150, sequenceid=336, filesize=12.0 K 2024-11-15T00:50:08,465 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 946ms, sequenceid=336, compaction requested=true 2024-11-15T00:50:08,465 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:08,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:08,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:08,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:08,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:50:08,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:08,465 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T00:50:08,465 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:08,468 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:08,470 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:08,470 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/A is initiating minor compaction (all files) 2024-11-15T00:50:08,470 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/A in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:08,471 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ceb15db3396b4f66a6fe9c3a18e52ee5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c0fcab67774a4259a6203bf9062232a1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/cc36814be8084bd49be4eb3a6d79becd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/749ec543a222447c917e154e566e6d4c] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=139.7 K 2024-11-15T00:50:08,471 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:08,471 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ceb15db3396b4f66a6fe9c3a18e52ee5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c0fcab67774a4259a6203bf9062232a1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/cc36814be8084bd49be4eb3a6d79becd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/749ec543a222447c917e154e566e6d4c] 2024-11-15T00:50:08,471 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ceb15db3396b4f66a6fe9c3a18e52ee5, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731631803734 2024-11-15T00:50:08,471 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c0fcab67774a4259a6203bf9062232a1, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731631803945 2024-11-15T00:50:08,473 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:08,473 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/C is initiating minor compaction (all files) 2024-11-15T00:50:08,473 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/C in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:08,473 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/3fe2715842624d75936aebd11b59d638, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/fce979540b29421d89daf8c637638859, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a344043eb80341d3b4b783582695b503, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/6330105da3614ce58b4a666b3bd3a901] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=48.7 K 2024-11-15T00:50:08,473 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting cc36814be8084bd49be4eb3a6d79becd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631805198 2024-11-15T00:50:08,474 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3fe2715842624d75936aebd11b59d638, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731631803734 2024-11-15T00:50:08,474 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 749ec543a222447c917e154e566e6d4c, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731631807457 2024-11-15T00:50:08,474 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting fce979540b29421d89daf8c637638859, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731631803945 2024-11-15T00:50:08,475 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a344043eb80341d3b4b783582695b503, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631805198 2024-11-15T00:50:08,475 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6330105da3614ce58b4a666b3bd3a901, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731631807509 2024-11-15T00:50:08,486 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:08,495 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#C#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:08,495 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/417a20fbebba4dbbbe55483a6879e842 is 50, key is test_row_0/C:col10/1731631807517/Put/seqid=0 2024-11-15T00:50:08,501 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115f2954d0414184b3094cbec64ee7b199a_d7dd7a757dbb0ea573f9e0b990dd7e24 store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:08,503 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115f2954d0414184b3094cbec64ee7b199a_d7dd7a757dbb0ea573f9e0b990dd7e24, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:08,504 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f2954d0414184b3094cbec64ee7b199a_d7dd7a757dbb0ea573f9e0b990dd7e24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:08,516 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742211_1387 (size=4469) 2024-11-15T00:50:08,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742210_1386 (size=13119) 2024-11-15T00:50:08,877 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:08,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=167.72 KB heapSize=440.20 KB 2024-11-15T00:50:08,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:08,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-15T00:50:08,878 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:08,879 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:08,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-15T00:50:08,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T00:50:08,882 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:08,882 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:08,883 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:08,886 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156a5414f9cd5a4d43bba93fad826dc680_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631808875/Put/seqid=0 2024-11-15T00:50:08,895 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742212_1388 (size=17534) 2024-11-15T00:50:08,896 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:08,900 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156a5414f9cd5a4d43bba93fad826dc680_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156a5414f9cd5a4d43bba93fad826dc680_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:08,903 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/6ea314b0102e43d38c5ae461311dca06, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:08,903 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/6ea314b0102e43d38c5ae461311dca06 is 175, key is test_row_0/A:col10/1731631808875/Put/seqid=0 2024-11-15T00:50:08,914 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:08,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631868904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:08,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 147 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631868904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:08,915 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:08,917 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#A#compaction#316 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:08,917 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/5f65cf5a651142ba963e1a769615dfb6 is 175, key is test_row_0/A:col10/1731631807517/Put/seqid=0 2024-11-15T00:50:08,921 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/417a20fbebba4dbbbe55483a6879e842 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/417a20fbebba4dbbbe55483a6879e842 2024-11-15T00:50:08,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742213_1389 (size=48639) 2024-11-15T00:50:08,922 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=364, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/6ea314b0102e43d38c5ae461311dca06 2024-11-15T00:50:08,931 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/C of d7dd7a757dbb0ea573f9e0b990dd7e24 into 417a20fbebba4dbbbe55483a6879e842(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:08,931 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:08,931 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/C, priority=12, startTime=1731631808465; duration=0sec 2024-11-15T00:50:08,931 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:08,931 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:C 2024-11-15T00:50:08,932 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:08,934 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49886 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:08,934 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/B is initiating minor compaction (all files) 2024-11-15T00:50:08,934 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/B in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:08,934 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/eacfd2b0797247b397e7627345bb751b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/d7378557849d4ed0b1cf01ef79542177, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7d5c3dc51a8b4b8ca98c560bc51f3d22, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/5c32adf0572d4a54a12f308967c80041] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=48.7 K 2024-11-15T00:50:08,934 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting eacfd2b0797247b397e7627345bb751b, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=285, earliestPutTs=1731631803734 2024-11-15T00:50:08,935 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7378557849d4ed0b1cf01ef79542177, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=313, earliestPutTs=1731631803945 2024-11-15T00:50:08,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742214_1390 (size=32073) 2024-11-15T00:50:08,938 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d5c3dc51a8b4b8ca98c560bc51f3d22, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631805198 2024-11-15T00:50:08,938 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c32adf0572d4a54a12f308967c80041, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731631807509 2024-11-15T00:50:08,943 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/5f65cf5a651142ba963e1a769615dfb6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/5f65cf5a651142ba963e1a769615dfb6 2024-11-15T00:50:08,948 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/A of d7dd7a757dbb0ea573f9e0b990dd7e24 into 5f65cf5a651142ba963e1a769615dfb6(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:08,948 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:08,948 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/A, priority=12, startTime=1731631808465; duration=0sec 2024-11-15T00:50:08,948 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:08,948 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:A 2024-11-15T00:50:08,951 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/7a7df459577e49c1b3dbb86675dd81f8 is 50, key is test_row_0/B:col10/1731631808875/Put/seqid=0 2024-11-15T00:50:08,954 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#B#compaction#320 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:08,954 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/04076d782c03460b83f16634462d68ec is 50, key is test_row_0/B:col10/1731631807517/Put/seqid=0 2024-11-15T00:50:08,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742215_1391 (size=12301) 2024-11-15T00:50:08,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T00:50:08,988 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742216_1392 (size=13119) 2024-11-15T00:50:09,020 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:09,020 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631869017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:09,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631869020, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,021 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:09,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T00:50:09,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:09,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:09,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:09,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:09,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:09,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:09,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T00:50:09,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:09,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:09,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:09,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:09,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:09,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:09,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T00:50:09,235 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:09,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631869227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:09,236 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:09,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631869227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:09,236 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,237 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:09,344 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T00:50:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:09,344 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] handler.RSProcedureHandler(58): pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:09,344 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=110 java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:09,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=110 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:09,374 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/7a7df459577e49c1b3dbb86675dd81f8 2024-11-15T00:50:09,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/777821d4a64248ed84a911aaf14bb454 is 50, key is test_row_0/C:col10/1731631808875/Put/seqid=0 2024-11-15T00:50:09,398 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/04076d782c03460b83f16634462d68ec as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/04076d782c03460b83f16634462d68ec 2024-11-15T00:50:09,406 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/B of d7dd7a757dbb0ea573f9e0b990dd7e24 into 04076d782c03460b83f16634462d68ec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:09,406 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:09,406 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/B, priority=12, startTime=1731631808465; duration=0sec 2024-11-15T00:50:09,406 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:09,408 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:B 2024-11-15T00:50:09,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742217_1393 (size=12301) 2024-11-15T00:50:09,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=364 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/777821d4a64248ed84a911aaf14bb454 2024-11-15T00:50:09,432 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/6ea314b0102e43d38c5ae461311dca06 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/6ea314b0102e43d38c5ae461311dca06 2024-11-15T00:50:09,450 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/6ea314b0102e43d38c5ae461311dca06, entries=250, sequenceid=364, filesize=47.5 K 2024-11-15T00:50:09,451 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/7a7df459577e49c1b3dbb86675dd81f8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7a7df459577e49c1b3dbb86675dd81f8 2024-11-15T00:50:09,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7a7df459577e49c1b3dbb86675dd81f8, entries=150, sequenceid=364, filesize=12.0 K 2024-11-15T00:50:09,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/777821d4a64248ed84a911aaf14bb454 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/777821d4a64248ed84a911aaf14bb454 2024-11-15T00:50:09,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/777821d4a64248ed84a911aaf14bb454, entries=150, sequenceid=364, filesize=12.0 K 2024-11-15T00:50:09,466 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 589ms, sequenceid=364, compaction requested=false 2024-11-15T00:50:09,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:09,497 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-15T00:50:09,497 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:09,498 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T00:50:09,511 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152b299f6ede69403ca5bbc9d92a6905fe_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631808901/Put/seqid=0 2024-11-15T00:50:09,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T00:50:09,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742218_1394 (size=12454) 2024-11-15T00:50:09,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:09,550 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411152b299f6ede69403ca5bbc9d92a6905fe_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152b299f6ede69403ca5bbc9d92a6905fe_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:09,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/7673f789272f47a49db5a014dd35f6e1, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:09,552 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/7673f789272f47a49db5a014dd35f6e1 is 175, key is test_row_0/A:col10/1731631808901/Put/seqid=0 2024-11-15T00:50:09,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:09,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. as already flushing 2024-11-15T00:50:09,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742219_1395 (size=31255) 2024-11-15T00:50:09,580 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=375, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/7673f789272f47a49db5a014dd35f6e1 2024-11-15T00:50:09,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/623e74b20e364fdcb843621762fa8db5 is 50, key is test_row_0/B:col10/1731631808901/Put/seqid=0 2024-11-15T00:50:09,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742220_1396 (size=12301) 2024-11-15T00:50:09,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:09,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631869650, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,652 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:09,659 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:09,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 184 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631869651, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,659 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:09,750 INFO [Thread-1357 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1357. 2024-11-15T00:50:09,750 DEBUG [Thread-1357 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:09,750 DEBUG [Thread-1357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:09,750 DEBUG [Thread-1357 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:09,750 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:09,753 INFO [Thread-1359 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1359. 2024-11-15T00:50:09,753 DEBUG [Thread-1359 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:09,753 DEBUG [Thread-1359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:09,753 DEBUG [Thread-1359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:09,753 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:09,754 INFO [Thread-1361 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1361. 2024-11-15T00:50:09,754 DEBUG [Thread-1361 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:09,754 DEBUG [Thread-1361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:09,754 DEBUG [Thread-1361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:09,754 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:09,757 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:09,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631869756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:09,760 INFO [Thread-1353 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1353. 2024-11-15T00:50:09,760 DEBUG [Thread-1353 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:09,760 DEBUG [Thread-1353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:09,760 DEBUG [Thread-1353 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:09,760 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:09,761 INFO [Thread-1355 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1355. 2024-11-15T00:50:09,761 DEBUG [Thread-1355 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:09,761 DEBUG [Thread-1355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:09,761 DEBUG [Thread-1355 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:09,761 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:09,767 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:09,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631869767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,767 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:09,967 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:09,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631869967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,968 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:09,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:09,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631869976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:09,977 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:09,994 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/623e74b20e364fdcb843621762fa8db5 2024-11-15T00:50:10,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/e7afb5895d2b43a5a7a4c8417abccf0b is 50, key is test_row_0/C:col10/1731631808901/Put/seqid=0 2024-11-15T00:50:10,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742221_1397 (size=12301) 2024-11-15T00:50:10,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T00:50:10,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:10,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38050 deadline: 1731631870277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:10,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:10,287 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:10,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:38020 deadline: 1731631870287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:10,288 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:10,404 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=375 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/e7afb5895d2b43a5a7a4c8417abccf0b 2024-11-15T00:50:10,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/7673f789272f47a49db5a014dd35f6e1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7673f789272f47a49db5a014dd35f6e1 2024-11-15T00:50:10,412 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7673f789272f47a49db5a014dd35f6e1, entries=150, sequenceid=375, filesize=30.5 K 2024-11-15T00:50:10,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/623e74b20e364fdcb843621762fa8db5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/623e74b20e364fdcb843621762fa8db5 2024-11-15T00:50:10,416 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/623e74b20e364fdcb843621762fa8db5, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T00:50:10,417 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/e7afb5895d2b43a5a7a4c8417abccf0b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/e7afb5895d2b43a5a7a4c8417abccf0b 2024-11-15T00:50:10,420 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/e7afb5895d2b43a5a7a4c8417abccf0b, entries=150, sequenceid=375, filesize=12.0 K 2024-11-15T00:50:10,421 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=167.72 KB/171750 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 924ms, sequenceid=375, compaction requested=true 2024-11-15T00:50:10,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:10,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:10,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-15T00:50:10,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-15T00:50:10,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-15T00:50:10,424 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5400 sec 2024-11-15T00:50:10,427 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.5460 sec 2024-11-15T00:50:10,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:10,789 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:10,789 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=174.43 KB heapSize=457.78 KB 2024-11-15T00:50:10,789 INFO [Thread-1350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1350. 2024-11-15T00:50:10,789 DEBUG [Thread-1350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:10,789 DEBUG [Thread-1350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:10,789 DEBUG [Thread-1350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:10,789 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:10,793 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156a21bc63c30f405ba4b156d034770a19_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631809629/Put/seqid=0 2024-11-15T00:50:10,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742222_1398 (size=12454) 2024-11-15T00:50:10,797 INFO [Thread-1344 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1344. 2024-11-15T00:50:10,797 DEBUG [Thread-1344 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:10,798 DEBUG [Thread-1344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:10,798 DEBUG [Thread-1344 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:10,798 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:11,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-15T00:50:11,037 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:11,268 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:11,276 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411156a21bc63c30f405ba4b156d034770a19_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156a21bc63c30f405ba4b156d034770a19_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:11,277 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e66f32fb4d844dc1ae2b38193fbf263e, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:11,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e66f32fb4d844dc1ae2b38193fbf263e is 175, key is test_row_0/A:col10/1731631809629/Put/seqid=0 2024-11-15T00:50:11,279 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742223_1399 (size=31255) 2024-11-15T00:50:11,681 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=404, memsize=58.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e66f32fb4d844dc1ae2b38193fbf263e 2024-11-15T00:50:11,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/427b982c09694cac9aa3cddf0953b2a4 is 50, key is test_row_0/B:col10/1731631809629/Put/seqid=0 2024-11-15T00:50:11,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742224_1400 (size=12301) 2024-11-15T00:50:12,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/427b982c09694cac9aa3cddf0953b2a4 2024-11-15T00:50:12,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/aef85701d1154d1d8e91b51fafe64485 is 50, key is test_row_0/C:col10/1731631809629/Put/seqid=0 2024-11-15T00:50:12,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742225_1401 (size=12301) 2024-11-15T00:50:12,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=58.14 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/aef85701d1154d1d8e91b51fafe64485 2024-11-15T00:50:12,512 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e66f32fb4d844dc1ae2b38193fbf263e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e66f32fb4d844dc1ae2b38193fbf263e 2024-11-15T00:50:12,515 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e66f32fb4d844dc1ae2b38193fbf263e, entries=150, sequenceid=404, filesize=30.5 K 2024-11-15T00:50:12,516 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/427b982c09694cac9aa3cddf0953b2a4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/427b982c09694cac9aa3cddf0953b2a4 2024-11-15T00:50:12,520 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/427b982c09694cac9aa3cddf0953b2a4, entries=150, sequenceid=404, filesize=12.0 K 2024-11-15T00:50:12,521 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/aef85701d1154d1d8e91b51fafe64485 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/aef85701d1154d1d8e91b51fafe64485 2024-11-15T00:50:12,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/aef85701d1154d1d8e91b51fafe64485, entries=150, sequenceid=404, filesize=12.0 K 2024-11-15T00:50:12,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~174.43 KB/178620, heapSize ~457.73 KB/468720, currentSize=6.71 KB/6870 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 1736ms, sequenceid=404, compaction requested=true 2024-11-15T00:50:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:12,525 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:12,525 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:12,525 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store d7dd7a757dbb0ea573f9e0b990dd7e24:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:12,526 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:12,526 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 143222 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:12,526 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:12,526 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/A is initiating minor compaction (all files) 2024-11-15T00:50:12,526 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/B is initiating minor compaction (all files) 2024-11-15T00:50:12,526 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/A in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:12,527 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/B in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:12,527 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/5f65cf5a651142ba963e1a769615dfb6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/6ea314b0102e43d38c5ae461311dca06, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7673f789272f47a49db5a014dd35f6e1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e66f32fb4d844dc1ae2b38193fbf263e] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=139.9 K 2024-11-15T00:50:12,527 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/04076d782c03460b83f16634462d68ec, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7a7df459577e49c1b3dbb86675dd81f8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/623e74b20e364fdcb843621762fa8db5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/427b982c09694cac9aa3cddf0953b2a4] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=48.8 K 2024-11-15T00:50:12,527 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:12,527 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/5f65cf5a651142ba963e1a769615dfb6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/6ea314b0102e43d38c5ae461311dca06, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7673f789272f47a49db5a014dd35f6e1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e66f32fb4d844dc1ae2b38193fbf263e] 2024-11-15T00:50:12,527 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 04076d782c03460b83f16634462d68ec, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731631807509 2024-11-15T00:50:12,527 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5f65cf5a651142ba963e1a769615dfb6, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731631807509 2024-11-15T00:50:12,527 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7a7df459577e49c1b3dbb86675dd81f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731631807678 2024-11-15T00:50:12,527 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6ea314b0102e43d38c5ae461311dca06, keycount=250, bloomtype=ROW, size=47.5 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731631807646 2024-11-15T00:50:12,527 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 623e74b20e364fdcb843621762fa8db5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731631808891 2024-11-15T00:50:12,528 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7673f789272f47a49db5a014dd35f6e1, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731631808891 2024-11-15T00:50:12,528 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 427b982c09694cac9aa3cddf0953b2a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731631809629 2024-11-15T00:50:12,528 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e66f32fb4d844dc1ae2b38193fbf263e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731631809629 2024-11-15T00:50:12,539 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#B#compaction#328 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:12,539 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/71069b2d8bd744c2bbeeb93ddeb682ef is 50, key is test_row_0/B:col10/1731631809629/Put/seqid=0 2024-11-15T00:50:12,540 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:12,542 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115f4935e3d46fa44ac9a20adebfbd34a1b_d7dd7a757dbb0ea573f9e0b990dd7e24 store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:12,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742226_1402 (size=13255) 2024-11-15T00:50:12,558 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/71069b2d8bd744c2bbeeb93ddeb682ef as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/71069b2d8bd744c2bbeeb93ddeb682ef 2024-11-15T00:50:12,563 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/B of d7dd7a757dbb0ea573f9e0b990dd7e24 into 71069b2d8bd744c2bbeeb93ddeb682ef(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:12,563 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:12,563 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/B, priority=12, startTime=1731631812525; duration=0sec 2024-11-15T00:50:12,563 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:12,563 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:B 2024-11-15T00:50:12,563 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:12,565 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50022 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:12,565 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): d7dd7a757dbb0ea573f9e0b990dd7e24/C is initiating minor compaction (all files) 2024-11-15T00:50:12,565 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of d7dd7a757dbb0ea573f9e0b990dd7e24/C in TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:12,565 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/417a20fbebba4dbbbe55483a6879e842, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/777821d4a64248ed84a911aaf14bb454, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/e7afb5895d2b43a5a7a4c8417abccf0b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/aef85701d1154d1d8e91b51fafe64485] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp, totalSize=48.8 K 2024-11-15T00:50:12,566 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 417a20fbebba4dbbbe55483a6879e842, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=336, earliestPutTs=1731631807509 2024-11-15T00:50:12,567 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 777821d4a64248ed84a911aaf14bb454, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=364, earliestPutTs=1731631807678 2024-11-15T00:50:12,567 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e7afb5895d2b43a5a7a4c8417abccf0b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=375, earliestPutTs=1731631808891 2024-11-15T00:50:12,568 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting aef85701d1154d1d8e91b51fafe64485, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731631809629 2024-11-15T00:50:12,582 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115f4935e3d46fa44ac9a20adebfbd34a1b_d7dd7a757dbb0ea573f9e0b990dd7e24, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:12,582 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f4935e3d46fa44ac9a20adebfbd34a1b_d7dd7a757dbb0ea573f9e0b990dd7e24 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:12,589 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742227_1403 (size=4469) 2024-11-15T00:50:12,591 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#A#compaction#329 average throughput is 0.48 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:12,591 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e40b4be7a9764b028beb0a6657a30f15 is 175, key is test_row_0/A:col10/1731631809629/Put/seqid=0 2024-11-15T00:50:12,601 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): d7dd7a757dbb0ea573f9e0b990dd7e24#C#compaction#330 average throughput is 0.39 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:12,602 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/bd84e5c96a424414bc08f1cac78e8ead is 50, key is test_row_0/C:col10/1731631809629/Put/seqid=0 2024-11-15T00:50:12,609 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742228_1404 (size=32209) 2024-11-15T00:50:12,613 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742229_1405 (size=13255) 2024-11-15T00:50:12,628 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/e40b4be7a9764b028beb0a6657a30f15 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e40b4be7a9764b028beb0a6657a30f15 2024-11-15T00:50:12,634 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/A of d7dd7a757dbb0ea573f9e0b990dd7e24 into e40b4be7a9764b028beb0a6657a30f15(size=31.5 K), total size for store is 31.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:12,634 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:12,634 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/A, priority=12, startTime=1731631812525; duration=0sec 2024-11-15T00:50:12,634 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:12,634 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:A 2024-11-15T00:50:12,782 INFO [Thread-1346 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1346. 2024-11-15T00:50:12,782 DEBUG [Thread-1346 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:12,782 DEBUG [Thread-1346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:12,782 DEBUG [Thread-1346 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:12,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:13,037 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/bd84e5c96a424414bc08f1cac78e8ead as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/bd84e5c96a424414bc08f1cac78e8ead 2024-11-15T00:50:13,053 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in d7dd7a757dbb0ea573f9e0b990dd7e24/C of d7dd7a757dbb0ea573f9e0b990dd7e24 into bd84e5c96a424414bc08f1cac78e8ead(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:13,053 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for d7dd7a757dbb0ea573f9e0b990dd7e24: 2024-11-15T00:50:13,053 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., storeName=d7dd7a757dbb0ea573f9e0b990dd7e24/C, priority=12, startTime=1731631812525; duration=0sec 2024-11-15T00:50:13,053 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:13,053 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: d7dd7a757dbb0ea573f9e0b990dd7e24:C 2024-11-15T00:50:14,819 INFO [Thread-1342 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1342. 2024-11-15T00:50:14,819 DEBUG [Thread-1342 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:14,819 DEBUG [Thread-1342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:14,820 DEBUG [Thread-1342 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:14,820 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:14,838 INFO [Thread-1348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1348. 2024-11-15T00:50:14,839 DEBUG [Thread-1348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:14,839 DEBUG [Thread-1348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:14,839 DEBUG [Thread-1348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:14,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 29 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 123 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 33 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 116 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1000 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2996 rows 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1009 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3021 rows 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1009 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3025 rows 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1013 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3036 rows 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1003 2024-11-15T00:50:14,839 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 3008 rows 2024-11-15T00:50:14,839 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:50:14,839 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:50:14,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:14,839 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:14,839 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:14,840 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:50:14,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:14,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:50:14,846 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631814845"}]},"ts":"1731631814845"} 2024-11-15T00:50:14,850 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:50:14,850 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:50:14,851 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:50:14,852 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, UNASSIGN}] 2024-11-15T00:50:14,854 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, UNASSIGN 2024-11-15T00:50:14,855 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=d7dd7a757dbb0ea573f9e0b990dd7e24, regionState=CLOSING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:14,857 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=113, ppid=112, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, UNASSIGN because future has completed 2024-11-15T00:50:14,857 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:50:14,857 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:50:14,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:50:15,010 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(122): Close d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:15,010 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:50:15,011 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1722): Closing d7dd7a757dbb0ea573f9e0b990dd7e24, disabling compactions & flushes 2024-11-15T00:50:15,011 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:15,011 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:15,011 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. after waiting 0 ms 2024-11-15T00:50:15,011 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:15,011 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(2902): Flushing d7dd7a757dbb0ea573f9e0b990dd7e24 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-15T00:50:15,017 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111532ac9abf5c1b4a72aad865a5472a549a_d7dd7a757dbb0ea573f9e0b990dd7e24 is 50, key is test_row_0/A:col10/1731631814837/Put/seqid=0 2024-11-15T00:50:15,023 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742230_1406 (size=12454) 2024-11-15T00:50:15,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:50:15,424 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:15,430 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111532ac9abf5c1b4a72aad865a5472a549a_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111532ac9abf5c1b4a72aad865a5472a549a_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:15,431 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f27ec98f08d0452697872c2ffc6b6e36, store: [table=TestAcidGuarantees family=A region=d7dd7a757dbb0ea573f9e0b990dd7e24] 2024-11-15T00:50:15,432 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f27ec98f08d0452697872c2ffc6b6e36 is 175, key is test_row_0/A:col10/1731631814837/Put/seqid=0 2024-11-15T00:50:15,442 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742231_1407 (size=31255) 2024-11-15T00:50:15,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:50:15,846 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=414, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f27ec98f08d0452697872c2ffc6b6e36 2024-11-15T00:50:15,866 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/d242b8fcf2cd4b2893af6409cde5e725 is 50, key is test_row_0/B:col10/1731631814837/Put/seqid=0 2024-11-15T00:50:15,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742232_1408 (size=12301) 2024-11-15T00:50:15,902 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/d242b8fcf2cd4b2893af6409cde5e725 2024-11-15T00:50:15,910 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/1459fae2692444dfafdb951d44d68e2a is 50, key is test_row_0/C:col10/1731631814837/Put/seqid=0 2024-11-15T00:50:15,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742233_1409 (size=12301) 2024-11-15T00:50:15,933 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=414 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/1459fae2692444dfafdb951d44d68e2a 2024-11-15T00:50:15,950 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/A/f27ec98f08d0452697872c2ffc6b6e36 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f27ec98f08d0452697872c2ffc6b6e36 2024-11-15T00:50:15,956 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f27ec98f08d0452697872c2ffc6b6e36, entries=150, sequenceid=414, filesize=30.5 K 2024-11-15T00:50:15,957 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/B/d242b8fcf2cd4b2893af6409cde5e725 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/d242b8fcf2cd4b2893af6409cde5e725 2024-11-15T00:50:15,975 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/d242b8fcf2cd4b2893af6409cde5e725, entries=150, sequenceid=414, filesize=12.0 K 2024-11-15T00:50:15,976 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/.tmp/C/1459fae2692444dfafdb951d44d68e2a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/1459fae2692444dfafdb951d44d68e2a 2024-11-15T00:50:15,986 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/1459fae2692444dfafdb951d44d68e2a, entries=150, sequenceid=414, filesize=12.0 K 2024-11-15T00:50:15,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:50:15,987 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 976ms, sequenceid=414, compaction requested=false 2024-11-15T00:50:15,987 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/181bbb6d89ca447abc6b6f5c5120f65b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f924e718f82a4822b0efa079cf2e9b56, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f44a633391e44903ad5514db925b8ea6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c8b1eb5560ab4bfe8bba94efad1e4161, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/77a86f6ff226430dad1b715da9f2bc9f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/295a26ed27a748cbbd872036a6103740, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2855952e06de4ac9af038b7bce0ced57, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e0944808dce945d1a1746bb1070d85d8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/d81e922fc25a48d2b57385497762e947, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7444c1de8a424f28a2f632b1d53b7601, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/bb19f02b773f4667aa16c3de90d8eec5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c75d62ce6d9b4646b378888e0eac48ea, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e42536a1cada4779a0b9c70207e6ccd0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/96b57c9041f54640bfe1517b21586710, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/81dba6b21bc9460590e4f2da9a889973, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ce1df1de587f4745b438e3d2b459dec4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/dd89145da4064eb58eccf587fa2b200c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f2d9ea92144b4e34a4b4b4ee3225f40c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/23e302648ea04b93a619348fd4975ffa, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2a5e6f564aaf4b64bce241da569964f2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ceb15db3396b4f66a6fe9c3a18e52ee5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c0fcab67774a4259a6203bf9062232a1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/cc36814be8084bd49be4eb3a6d79becd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/749ec543a222447c917e154e566e6d4c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/5f65cf5a651142ba963e1a769615dfb6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/6ea314b0102e43d38c5ae461311dca06, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7673f789272f47a49db5a014dd35f6e1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e66f32fb4d844dc1ae2b38193fbf263e] to archive 2024-11-15T00:50:15,988 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:50:15,990 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/181bbb6d89ca447abc6b6f5c5120f65b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/181bbb6d89ca447abc6b6f5c5120f65b 2024-11-15T00:50:15,991 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f924e718f82a4822b0efa079cf2e9b56 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f924e718f82a4822b0efa079cf2e9b56 2024-11-15T00:50:15,992 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f44a633391e44903ad5514db925b8ea6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f44a633391e44903ad5514db925b8ea6 2024-11-15T00:50:15,997 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c8b1eb5560ab4bfe8bba94efad1e4161 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c8b1eb5560ab4bfe8bba94efad1e4161 2024-11-15T00:50:15,998 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/77a86f6ff226430dad1b715da9f2bc9f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/77a86f6ff226430dad1b715da9f2bc9f 2024-11-15T00:50:15,999 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/295a26ed27a748cbbd872036a6103740 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/295a26ed27a748cbbd872036a6103740 2024-11-15T00:50:16,000 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2855952e06de4ac9af038b7bce0ced57 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2855952e06de4ac9af038b7bce0ced57 2024-11-15T00:50:16,002 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e0944808dce945d1a1746bb1070d85d8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e0944808dce945d1a1746bb1070d85d8 2024-11-15T00:50:16,003 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/d81e922fc25a48d2b57385497762e947 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/d81e922fc25a48d2b57385497762e947 2024-11-15T00:50:16,005 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7444c1de8a424f28a2f632b1d53b7601 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7444c1de8a424f28a2f632b1d53b7601 2024-11-15T00:50:16,006 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/bb19f02b773f4667aa16c3de90d8eec5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/bb19f02b773f4667aa16c3de90d8eec5 2024-11-15T00:50:16,008 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c75d62ce6d9b4646b378888e0eac48ea to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c75d62ce6d9b4646b378888e0eac48ea 2024-11-15T00:50:16,010 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e42536a1cada4779a0b9c70207e6ccd0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e42536a1cada4779a0b9c70207e6ccd0 2024-11-15T00:50:16,011 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/96b57c9041f54640bfe1517b21586710 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/96b57c9041f54640bfe1517b21586710 2024-11-15T00:50:16,012 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/81dba6b21bc9460590e4f2da9a889973 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/81dba6b21bc9460590e4f2da9a889973 2024-11-15T00:50:16,014 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ce1df1de587f4745b438e3d2b459dec4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ce1df1de587f4745b438e3d2b459dec4 2024-11-15T00:50:16,015 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/dd89145da4064eb58eccf587fa2b200c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/dd89145da4064eb58eccf587fa2b200c 2024-11-15T00:50:16,017 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f2d9ea92144b4e34a4b4b4ee3225f40c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f2d9ea92144b4e34a4b4b4ee3225f40c 2024-11-15T00:50:16,029 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/23e302648ea04b93a619348fd4975ffa to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/23e302648ea04b93a619348fd4975ffa 2024-11-15T00:50:16,048 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2a5e6f564aaf4b64bce241da569964f2 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/2a5e6f564aaf4b64bce241da569964f2 2024-11-15T00:50:16,060 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ceb15db3396b4f66a6fe9c3a18e52ee5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/ceb15db3396b4f66a6fe9c3a18e52ee5 2024-11-15T00:50:16,068 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c0fcab67774a4259a6203bf9062232a1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/c0fcab67774a4259a6203bf9062232a1 2024-11-15T00:50:16,070 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/cc36814be8084bd49be4eb3a6d79becd to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/cc36814be8084bd49be4eb3a6d79becd 2024-11-15T00:50:16,071 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/749ec543a222447c917e154e566e6d4c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/749ec543a222447c917e154e566e6d4c 2024-11-15T00:50:16,074 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/5f65cf5a651142ba963e1a769615dfb6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/5f65cf5a651142ba963e1a769615dfb6 2024-11-15T00:50:16,076 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/6ea314b0102e43d38c5ae461311dca06 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/6ea314b0102e43d38c5ae461311dca06 2024-11-15T00:50:16,086 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7673f789272f47a49db5a014dd35f6e1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/7673f789272f47a49db5a014dd35f6e1 2024-11-15T00:50:16,091 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e66f32fb4d844dc1ae2b38193fbf263e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e66f32fb4d844dc1ae2b38193fbf263e 2024-11-15T00:50:16,098 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/db340441f63f4f5ea916db457ae9cf3d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/97350c9869434980adec47f3d18b4b96, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/00997875cc3647499c3c4d12dd0e4500, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1c3ff85f555f4c2fa994e179fe2e0b38, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6d204828d520441fa7bffa2365059ad4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a7811d8e350f405cafc59fac5fbf3c28, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1bf7711efd2941f69fccc58990e5ddc4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/2da5f7c528784928aea7e18d441ea6e0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/05b9974696364660a9382aa69e4e49cc, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/34bf5399c0ae4a199e8341035e9ae564, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6005653e88ca4151912589fa68030303, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a05a66600e684669841ddf5ba1dd3934, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/41a22cfe3b544130912fa3d794d85721, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/bffd343afc63492ba438d783567808f2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/e75f262cda0d4d598703d09c7dc2fb72, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ee91687e50cc42ac9f211dfc4e620309, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/0486aa721db44080a51f818e9188fcef, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/859d2b5688aa4d96b883711ad24da937, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ac6c2042f8394bc28f0be3429ad67564, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/eacfd2b0797247b397e7627345bb751b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/13db97a681734e7dae9cbe69218b4af5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/d7378557849d4ed0b1cf01ef79542177, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7d5c3dc51a8b4b8ca98c560bc51f3d22, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/04076d782c03460b83f16634462d68ec, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/5c32adf0572d4a54a12f308967c80041, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7a7df459577e49c1b3dbb86675dd81f8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/623e74b20e364fdcb843621762fa8db5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/427b982c09694cac9aa3cddf0953b2a4] to archive 2024-11-15T00:50:16,100 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:50:16,105 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/db340441f63f4f5ea916db457ae9cf3d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/db340441f63f4f5ea916db457ae9cf3d 2024-11-15T00:50:16,107 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/97350c9869434980adec47f3d18b4b96 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/97350c9869434980adec47f3d18b4b96 2024-11-15T00:50:16,108 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/00997875cc3647499c3c4d12dd0e4500 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/00997875cc3647499c3c4d12dd0e4500 2024-11-15T00:50:16,109 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1c3ff85f555f4c2fa994e179fe2e0b38 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1c3ff85f555f4c2fa994e179fe2e0b38 2024-11-15T00:50:16,111 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6d204828d520441fa7bffa2365059ad4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6d204828d520441fa7bffa2365059ad4 2024-11-15T00:50:16,113 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a7811d8e350f405cafc59fac5fbf3c28 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a7811d8e350f405cafc59fac5fbf3c28 2024-11-15T00:50:16,114 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1bf7711efd2941f69fccc58990e5ddc4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/1bf7711efd2941f69fccc58990e5ddc4 2024-11-15T00:50:16,115 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/2da5f7c528784928aea7e18d441ea6e0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/2da5f7c528784928aea7e18d441ea6e0 2024-11-15T00:50:16,117 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/05b9974696364660a9382aa69e4e49cc to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/05b9974696364660a9382aa69e4e49cc 2024-11-15T00:50:16,118 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/34bf5399c0ae4a199e8341035e9ae564 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/34bf5399c0ae4a199e8341035e9ae564 2024-11-15T00:50:16,119 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6005653e88ca4151912589fa68030303 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/6005653e88ca4151912589fa68030303 2024-11-15T00:50:16,121 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a05a66600e684669841ddf5ba1dd3934 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/a05a66600e684669841ddf5ba1dd3934 2024-11-15T00:50:16,126 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/41a22cfe3b544130912fa3d794d85721 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/41a22cfe3b544130912fa3d794d85721 2024-11-15T00:50:16,127 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/bffd343afc63492ba438d783567808f2 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/bffd343afc63492ba438d783567808f2 2024-11-15T00:50:16,128 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/e75f262cda0d4d598703d09c7dc2fb72 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/e75f262cda0d4d598703d09c7dc2fb72 2024-11-15T00:50:16,130 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ee91687e50cc42ac9f211dfc4e620309 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ee91687e50cc42ac9f211dfc4e620309 2024-11-15T00:50:16,131 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/0486aa721db44080a51f818e9188fcef to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/0486aa721db44080a51f818e9188fcef 2024-11-15T00:50:16,132 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/859d2b5688aa4d96b883711ad24da937 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/859d2b5688aa4d96b883711ad24da937 2024-11-15T00:50:16,133 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ac6c2042f8394bc28f0be3429ad67564 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/ac6c2042f8394bc28f0be3429ad67564 2024-11-15T00:50:16,137 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/eacfd2b0797247b397e7627345bb751b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/eacfd2b0797247b397e7627345bb751b 2024-11-15T00:50:16,139 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/13db97a681734e7dae9cbe69218b4af5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/13db97a681734e7dae9cbe69218b4af5 2024-11-15T00:50:16,141 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/d7378557849d4ed0b1cf01ef79542177 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/d7378557849d4ed0b1cf01ef79542177 2024-11-15T00:50:16,142 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7d5c3dc51a8b4b8ca98c560bc51f3d22 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7d5c3dc51a8b4b8ca98c560bc51f3d22 2024-11-15T00:50:16,148 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/04076d782c03460b83f16634462d68ec to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/04076d782c03460b83f16634462d68ec 2024-11-15T00:50:16,150 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/5c32adf0572d4a54a12f308967c80041 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/5c32adf0572d4a54a12f308967c80041 2024-11-15T00:50:16,152 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7a7df459577e49c1b3dbb86675dd81f8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/7a7df459577e49c1b3dbb86675dd81f8 2024-11-15T00:50:16,158 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/623e74b20e364fdcb843621762fa8db5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/623e74b20e364fdcb843621762fa8db5 2024-11-15T00:50:16,163 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/427b982c09694cac9aa3cddf0953b2a4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/427b982c09694cac9aa3cddf0953b2a4 2024-11-15T00:50:16,180 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/0838651b3de04ecbb746dff0c8db3164, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2d0fe0e3ec64ec296021ff85f97bfe0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/c5d1a45b378f4f76bb678580b10468a6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/08b080c024cc49a08af8ac670a8e7238, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/90d2fac986054d4cb04c2b4bd5d9fb8f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a7056081830e49a684f448f0313bafbf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/254f6ae5608b4321844dad0fd6c339c5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/5580aa2e3b7249efbeb344bfcabbca0a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17ec2a33a6884e84a0ea8d6eaa0d235e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/36c0b0ca18274ada9cd4def07ca7b43b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/311d01c593b6426980a8fc1acc2285ba, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cbfa82c85535432da6f9ad90cc66f5d2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/20211d1c37d541db8edda0d3475f1b82, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cab7708decb148ec970d2df82d0a228f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a0eb80d4c39b407eac571afc6110458a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/61043d7ce37d4f2580057b57ff9549b3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/7299f9113bcd4d2fb03067fb941f5311, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17b3e344fc614bb8ba4c62f99066ad01, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/79eea3377fa04d4787acb221447ce48b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/3fe2715842624d75936aebd11b59d638, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2bd99c801474d70b6af8ab2d1a22182, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/fce979540b29421d89daf8c637638859, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a344043eb80341d3b4b783582695b503, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/417a20fbebba4dbbbe55483a6879e842, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/6330105da3614ce58b4a666b3bd3a901, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/777821d4a64248ed84a911aaf14bb454, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/e7afb5895d2b43a5a7a4c8417abccf0b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/aef85701d1154d1d8e91b51fafe64485] to archive 2024-11-15T00:50:16,181 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:50:16,183 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/0838651b3de04ecbb746dff0c8db3164 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/0838651b3de04ecbb746dff0c8db3164 2024-11-15T00:50:16,184 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2d0fe0e3ec64ec296021ff85f97bfe0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2d0fe0e3ec64ec296021ff85f97bfe0 2024-11-15T00:50:16,185 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/c5d1a45b378f4f76bb678580b10468a6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/c5d1a45b378f4f76bb678580b10468a6 2024-11-15T00:50:16,186 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/08b080c024cc49a08af8ac670a8e7238 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/08b080c024cc49a08af8ac670a8e7238 2024-11-15T00:50:16,187 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/90d2fac986054d4cb04c2b4bd5d9fb8f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/90d2fac986054d4cb04c2b4bd5d9fb8f 2024-11-15T00:50:16,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a7056081830e49a684f448f0313bafbf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a7056081830e49a684f448f0313bafbf 2024-11-15T00:50:16,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/254f6ae5608b4321844dad0fd6c339c5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/254f6ae5608b4321844dad0fd6c339c5 2024-11-15T00:50:16,190 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/5580aa2e3b7249efbeb344bfcabbca0a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/5580aa2e3b7249efbeb344bfcabbca0a 2024-11-15T00:50:16,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17ec2a33a6884e84a0ea8d6eaa0d235e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17ec2a33a6884e84a0ea8d6eaa0d235e 2024-11-15T00:50:16,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/36c0b0ca18274ada9cd4def07ca7b43b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/36c0b0ca18274ada9cd4def07ca7b43b 2024-11-15T00:50:16,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/311d01c593b6426980a8fc1acc2285ba to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/311d01c593b6426980a8fc1acc2285ba 2024-11-15T00:50:16,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cbfa82c85535432da6f9ad90cc66f5d2 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cbfa82c85535432da6f9ad90cc66f5d2 2024-11-15T00:50:16,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/20211d1c37d541db8edda0d3475f1b82 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/20211d1c37d541db8edda0d3475f1b82 2024-11-15T00:50:16,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cab7708decb148ec970d2df82d0a228f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/cab7708decb148ec970d2df82d0a228f 2024-11-15T00:50:16,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a0eb80d4c39b407eac571afc6110458a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a0eb80d4c39b407eac571afc6110458a 2024-11-15T00:50:16,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/61043d7ce37d4f2580057b57ff9549b3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/61043d7ce37d4f2580057b57ff9549b3 2024-11-15T00:50:16,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/7299f9113bcd4d2fb03067fb941f5311 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/7299f9113bcd4d2fb03067fb941f5311 2024-11-15T00:50:16,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17b3e344fc614bb8ba4c62f99066ad01 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/17b3e344fc614bb8ba4c62f99066ad01 2024-11-15T00:50:16,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/79eea3377fa04d4787acb221447ce48b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/79eea3377fa04d4787acb221447ce48b 2024-11-15T00:50:16,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/3fe2715842624d75936aebd11b59d638 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/3fe2715842624d75936aebd11b59d638 2024-11-15T00:50:16,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2bd99c801474d70b6af8ab2d1a22182 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a2bd99c801474d70b6af8ab2d1a22182 2024-11-15T00:50:16,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/fce979540b29421d89daf8c637638859 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/fce979540b29421d89daf8c637638859 2024-11-15T00:50:16,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a344043eb80341d3b4b783582695b503 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/a344043eb80341d3b4b783582695b503 2024-11-15T00:50:16,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/417a20fbebba4dbbbe55483a6879e842 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/417a20fbebba4dbbbe55483a6879e842 2024-11-15T00:50:16,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/6330105da3614ce58b4a666b3bd3a901 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/6330105da3614ce58b4a666b3bd3a901 2024-11-15T00:50:16,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/777821d4a64248ed84a911aaf14bb454 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/777821d4a64248ed84a911aaf14bb454 2024-11-15T00:50:16,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/e7afb5895d2b43a5a7a4c8417abccf0b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/e7afb5895d2b43a5a7a4c8417abccf0b 2024-11-15T00:50:16,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/aef85701d1154d1d8e91b51fafe64485 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/aef85701d1154d1d8e91b51fafe64485 2024-11-15T00:50:16,213 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/recovered.edits/417.seqid, newMaxSeqId=417, maxSeqId=4 2024-11-15T00:50:16,214 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. 2024-11-15T00:50:16,214 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] regionserver.HRegion(1676): Region close journal for d7dd7a757dbb0ea573f9e0b990dd7e24: Waiting for close lock at 1731631815011Running coprocessor pre-close hooks at 1731631815011Disabling compacts and flushes for region at 1731631815011Disabling writes for close at 1731631815011Obtaining lock to block concurrent updates at 1731631815011Preparing flush snapshotting stores in d7dd7a757dbb0ea573f9e0b990dd7e24 at 1731631815011Finished memstore snapshotting TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1731631815011Flushing stores of TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24. at 1731631815012 (+1 ms)Flushing d7dd7a757dbb0ea573f9e0b990dd7e24/A: creating writer at 1731631815012Flushing mob file d7dd7a757dbb0ea573f9e0b990dd7e24/A: appending metadata at 1731631815017 (+5 ms)Flushing mob file d7dd7a757dbb0ea573f9e0b990dd7e24/A: closing flushed file at 1731631815017Flushing d7dd7a757dbb0ea573f9e0b990dd7e24/A: appending metadata at 1731631815431 (+414 ms)Flushing d7dd7a757dbb0ea573f9e0b990dd7e24/A: closing flushed file at 1731631815431Flushing d7dd7a757dbb0ea573f9e0b990dd7e24/B: creating writer at 1731631815854 (+423 ms)Flushing d7dd7a757dbb0ea573f9e0b990dd7e24/B: appending metadata at 1731631815865 (+11 ms)Flushing d7dd7a757dbb0ea573f9e0b990dd7e24/B: closing flushed file at 1731631815865Flushing d7dd7a757dbb0ea573f9e0b990dd7e24/C: creating writer at 1731631815906 (+41 ms)Flushing d7dd7a757dbb0ea573f9e0b990dd7e24/C: appending metadata at 1731631815909 (+3 ms)Flushing d7dd7a757dbb0ea573f9e0b990dd7e24/C: closing flushed file at 1731631815909Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@70f72bc5: reopening flushed file at 1731631815949 (+40 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@758fe583: reopening flushed file at 1731631815956 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@33942ffd: reopening flushed file at 1731631815975 (+19 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for d7dd7a757dbb0ea573f9e0b990dd7e24 in 976ms, sequenceid=414, compaction requested=false at 1731631815987 (+12 ms)Writing region close event to WAL at 1731631816210 (+223 ms)Running coprocessor post-close hooks at 1731631816214 (+4 ms)Closed at 1731631816214 2024-11-15T00:50:16,216 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=114}] handler.UnassignRegionHandler(157): Closed d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:16,216 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=113 updating hbase:meta row=d7dd7a757dbb0ea573f9e0b990dd7e24, regionState=CLOSED 2024-11-15T00:50:16,217 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=114, ppid=113, state=RUNNABLE, hasLock=false; CloseRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:50:16,220 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-15T00:50:16,221 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseRegionProcedure d7dd7a757dbb0ea573f9e0b990dd7e24, server=fb2f6cb7846a,42925,1731631698709 in 1.3610 sec 2024-11-15T00:50:16,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=113, resume processing ppid=112 2024-11-15T00:50:16,223 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, ppid=112, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=d7dd7a757dbb0ea573f9e0b990dd7e24, UNASSIGN in 1.3690 sec 2024-11-15T00:50:16,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-15T00:50:16,225 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3720 sec 2024-11-15T00:50:16,227 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631816226"}]},"ts":"1731631816226"} 2024-11-15T00:50:16,228 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:50:16,228 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:50:16,230 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3890 sec 2024-11-15T00:50:16,788 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:50:17,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-15T00:50:17,009 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:17,012 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:50:17,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:17,018 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=115, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:17,019 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=115, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:17,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T00:50:17,031 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,033 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/recovered.edits] 2024-11-15T00:50:17,037 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e40b4be7a9764b028beb0a6657a30f15 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/e40b4be7a9764b028beb0a6657a30f15 2024-11-15T00:50:17,039 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f27ec98f08d0452697872c2ffc6b6e36 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/A/f27ec98f08d0452697872c2ffc6b6e36 2024-11-15T00:50:17,042 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/71069b2d8bd744c2bbeeb93ddeb682ef to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/71069b2d8bd744c2bbeeb93ddeb682ef 2024-11-15T00:50:17,044 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/d242b8fcf2cd4b2893af6409cde5e725 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/B/d242b8fcf2cd4b2893af6409cde5e725 2024-11-15T00:50:17,048 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/1459fae2692444dfafdb951d44d68e2a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/1459fae2692444dfafdb951d44d68e2a 2024-11-15T00:50:17,050 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/bd84e5c96a424414bc08f1cac78e8ead to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/C/bd84e5c96a424414bc08f1cac78e8ead 2024-11-15T00:50:17,055 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/recovered.edits/417.seqid to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24/recovered.edits/417.seqid 2024-11-15T00:50:17,055 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,055 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:50:17,056 DEBUG [PEWorker-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:50:17,056 DEBUG [PEWorker-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T00:50:17,059 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115046c74685f764f088cdf9628a6b3a349_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115046c74685f764f088cdf9628a6b3a349_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,061 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151f9e72353a9e418c83b6d878c0ec15bf_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411151f9e72353a9e418c83b6d878c0ec15bf_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,062 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152b299f6ede69403ca5bbc9d92a6905fe_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152b299f6ede69403ca5bbc9d92a6905fe_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,063 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152bb3d36c60364f039b94378b06e50e1e_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411152bb3d36c60364f039b94378b06e50e1e_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,064 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153183449abeb246b391a9695f5098c6b2_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153183449abeb246b391a9695f5098c6b2_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,065 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111532ac9abf5c1b4a72aad865a5472a549a_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111532ac9abf5c1b4a72aad865a5472a549a_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,066 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115372136f5c4f748d595e17503c9f7cbc9_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115372136f5c4f748d595e17503c9f7cbc9_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,068 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a669c0c554949ad99c86b3cad42144d_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153a669c0c554949ad99c86b3cad42144d_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,069 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153e32254f7de145be9c6671126b651749_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411153e32254f7de145be9c6671126b651749_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,077 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154cee1aad0cd747c7a1d10c36e15d4b0c_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411154cee1aad0cd747c7a1d10c36e15d4b0c_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,082 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155deff2be8eb44af18dc9c0dc70d027db_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411155deff2be8eb44af18dc9c0dc70d027db_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,086 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156a21bc63c30f405ba4b156d034770a19_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156a21bc63c30f405ba4b156d034770a19_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,089 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156a5414f9cd5a4d43bba93fad826dc680_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411156a5414f9cd5a4d43bba93fad826dc680_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,091 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111581bea5a7797045cebe9b2574e9ea2c07_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111581bea5a7797045cebe9b2574e9ea2c07_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,094 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159a2c3b1aac5543af9dca5348939020a7_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159a2c3b1aac5543af9dca5348939020a7_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,095 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159ca878cf534a4fed87b7a54951e1c016_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411159ca878cf534a4fed87b7a54951e1c016_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,096 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115be357ce96b6f432cae47b23f86d5dacd_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115be357ce96b6f432cae47b23f86d5dacd_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,098 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0e574fba8864e7b98d0f04e03f19872_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c0e574fba8864e7b98d0f04e03f19872_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,102 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c5b81b11f4d449738a4f5e022f4c779a_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c5b81b11f4d449738a4f5e022f4c779a_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,105 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ccec9c83bc5d419e9d3b15fb48bacc5c_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ccec9c83bc5d419e9d3b15fb48bacc5c_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,109 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e6b44f85fe4f4be5adb8f650099a5386_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e6b44f85fe4f4be5adb8f650099a5386_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,111 DEBUG [PEWorker-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ec78759e219f42c991d6cb96ea534abc_d7dd7a757dbb0ea573f9e0b990dd7e24 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ec78759e219f42c991d6cb96ea534abc_d7dd7a757dbb0ea573f9e0b990dd7e24 2024-11-15T00:50:17,112 DEBUG [PEWorker-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:50:17,127 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=115, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:17,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T00:50:17,130 WARN [PEWorker-2 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:50:17,134 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:50:17,141 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=115, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:17,141 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:50:17,142 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631817141"}]},"ts":"9223372036854775807"} 2024-11-15T00:50:17,147 INFO [PEWorker-2 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:50:17,147 DEBUG [PEWorker-2 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => d7dd7a757dbb0ea573f9e0b990dd7e24, NAME => 'TestAcidGuarantees,,1731631787607.d7dd7a757dbb0ea573f9e0b990dd7e24.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:50:17,147 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:50:17,148 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631817147"}]},"ts":"9223372036854775807"} 2024-11-15T00:50:17,151 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:50:17,155 DEBUG [PEWorker-2 {}] procedure.DeleteTableProcedure(133): Finished pid=115, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:17,157 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 143 msec 2024-11-15T00:50:17,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=115 2024-11-15T00:50:17,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:50:17,337 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:17,358 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=241 (was 238) - Thread LEAK? -, OpenFileDescriptor=461 (was 449) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=977 (was 933) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=2673 (was 1758) - AvailableMemoryMB LEAK? - 2024-11-15T00:50:17,388 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241, OpenFileDescriptor=461, MaxFileDescriptor=1048576, SystemLoadAverage=977, ProcessCount=11, AvailableMemoryMB=2667 2024-11-15T00:50:17,391 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:50:17,391 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:50:17,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:17,410 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:50:17,410 DEBUG [PEWorker-4 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:17,411 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 116 2024-11-15T00:50:17,411 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:50:17,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-15T00:50:17,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742234_1410 (size=959) 2024-11-15T00:50:17,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-15T00:50:17,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-15T00:50:17,855 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd 2024-11-15T00:50:17,889 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742235_1411 (size=53) 2024-11-15T00:50:17,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:50:17,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 0015d049952a9227751d617c5d05c387, disabling compactions & flushes 2024-11-15T00:50:17,897 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:17,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:17,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. after waiting 0 ms 2024-11-15T00:50:17,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:17,897 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:17,897 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 0015d049952a9227751d617c5d05c387: Waiting for close lock at 1731631817897Disabling compacts and flushes for region at 1731631817897Disabling writes for close at 1731631817897Writing region close event to WAL at 1731631817897Closed at 1731631817897 2024-11-15T00:50:17,899 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:50:17,899 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631817899"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631817899"}]},"ts":"1731631817899"} 2024-11-15T00:50:17,905 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:50:17,914 INFO [PEWorker-4 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:50:17,915 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631817914"}]},"ts":"1731631817914"} 2024-11-15T00:50:17,918 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:50:17,918 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0015d049952a9227751d617c5d05c387, ASSIGN}] 2024-11-15T00:50:17,919 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0015d049952a9227751d617c5d05c387, ASSIGN 2024-11-15T00:50:17,921 INFO [PEWorker-1 {}] assignment.TransitRegionStateProcedure(269): Starting pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0015d049952a9227751d617c5d05c387, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,42925,1731631698709; forceNewPlan=false, retain=false 2024-11-15T00:50:18,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-15T00:50:18,075 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=0015d049952a9227751d617c5d05c387, regionState=OPENING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:18,079 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=117, ppid=116, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0015d049952a9227751d617c5d05c387, ASSIGN because future has completed 2024-11-15T00:50:18,080 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:50:18,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T00:50:18,248 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:18,248 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7752): Opening region: {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:50:18,250 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,250 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:50:18,250 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7794): checking encryption for 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,250 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(7797): checking classloading for 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,261 INFO [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,277 INFO [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0015d049952a9227751d617c5d05c387 columnFamilyName A 2024-11-15T00:50:18,277 DEBUG [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:18,279 INFO [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] regionserver.HStore(327): Store=0015d049952a9227751d617c5d05c387/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:50:18,279 INFO [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,281 INFO [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0015d049952a9227751d617c5d05c387 columnFamilyName B 2024-11-15T00:50:18,281 DEBUG [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:18,282 INFO [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] regionserver.HStore(327): Store=0015d049952a9227751d617c5d05c387/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:50:18,282 INFO [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,284 INFO [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 0015d049952a9227751d617c5d05c387 columnFamilyName C 2024-11-15T00:50:18,284 DEBUG [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:18,292 INFO [StoreOpener-0015d049952a9227751d617c5d05c387-1 {}] regionserver.HStore(327): Store=0015d049952a9227751d617c5d05c387/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:50:18,293 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1038): replaying wal for 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,293 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,299 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,301 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1048): stopping wal replay for 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,301 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1060): Cleaning up temporary data for 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,302 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:50:18,303 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1093): writing seq id for 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,309 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:50:18,312 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1114): Opened 0015d049952a9227751d617c5d05c387; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=60306260, jitterRate=-0.10136669874191284}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:50:18,312 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,313 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegion(1006): Region open journal for 0015d049952a9227751d617c5d05c387: Running coprocessor pre-open hook at 1731631818250Writing region info on filesystem at 1731631818250Initializing all the Stores at 1731631818253 (+3 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631818253Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631818260 (+7 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631818260Cleaning up temporary data from old regions at 1731631818301 (+41 ms)Running coprocessor post-open hooks at 1731631818312 (+11 ms)Region opened successfully at 1731631818313 (+1 ms) 2024-11-15T00:50:18,315 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., pid=118, masterSystemTime=1731631818240 2024-11-15T00:50:18,322 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:18,322 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=118}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:18,322 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=117 updating hbase:meta row=0015d049952a9227751d617c5d05c387, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:18,333 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=118, ppid=117, state=RUNNABLE, hasLock=false; OpenRegionProcedure 0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:50:18,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=118, resume processing ppid=117 2024-11-15T00:50:18,351 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, ppid=117, state=SUCCESS, hasLock=false; OpenRegionProcedure 0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 in 262 msec 2024-11-15T00:50:18,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=117, resume processing ppid=116 2024-11-15T00:50:18,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, ppid=116, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0015d049952a9227751d617c5d05c387, ASSIGN in 433 msec 2024-11-15T00:50:18,357 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:50:18,357 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631818357"}]},"ts":"1731631818357"} 2024-11-15T00:50:18,364 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:50:18,365 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=116, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:50:18,368 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 974 msec 2024-11-15T00:50:18,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=116 2024-11-15T00:50:18,547 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:18,548 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@47e1a424, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,548 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,549 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,550 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ff0cc44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,550 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,551 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,552 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37574, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,555 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ed2d18f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,556 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,558 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,560 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,564 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:50:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,46397,1731631697729 2024-11-15T00:50:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5d71964b 2024-11-15T00:50:18,566 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:50:18,569 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37576, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:50:18,580 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407ebc73, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,581 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,581 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,588 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,592 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@406b80b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,593 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,595 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37600, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,596 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@300d33ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,600 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5159e3f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,601 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,601 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,602 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,602 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@46fbf6f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,603 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,604 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37630, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,605 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@e20e531, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,608 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1062bfdd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,608 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,609 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,613 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23213a13, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,616 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,618 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37646, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,619 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@390153d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,621 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4d7232e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,621 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,621 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,625 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a0e8688, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,630 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,634 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3f074898, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,635 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39880b68, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,635 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,635 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,637 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,640 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4aadd4ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,641 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,642 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,648 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3c26d2fd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,652 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4cb46e8a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,653 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,653 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,657 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,660 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@63dc2159, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,661 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,663 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,663 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@755410e0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,664 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1593fd05, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,664 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,664 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,667 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e315cd5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,667 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,668 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,669 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,670 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42cb3148, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,670 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@729114ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,670 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,671 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,672 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,672 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@226b5665, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,673 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,677 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37722, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,681 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1883511f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,685 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139f4c00, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,685 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,685 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,689 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@568737b6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,691 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,693 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37752, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,694 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1fc8d529, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,700 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14b03bcc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,700 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:18,701 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:18,703 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@758ffad5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:18,704 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:18,708 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:37778, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:18,709 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@79969e0b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:18,717 DEBUG [Thread-1859 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,721 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,728 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,732 DEBUG [Thread-1878 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,735 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:18,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees 2024-11-15T00:50:18,736 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:50:18,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:50:18,738 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:18,740 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,741 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,745 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56210, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,745 DEBUG [Thread-1863 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,745 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=119, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:18,745 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:18,746 DEBUG [Thread-1863 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,746 DEBUG [Thread-1863 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,746 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:50:18,749 DEBUG [Thread-1872 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,750 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,750 DEBUG [Thread-1870 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,751 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56222, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,751 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,752 DEBUG [Thread-1865 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:50:18,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,753 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,753 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56236, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,754 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56256, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,754 DEBUG [Thread-1876 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,755 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:50:18,756 DEBUG [Thread-1876 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,756 DEBUG [Thread-1876 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,756 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:50:18,757 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56264, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:50:18,761 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:50:18,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:18,764 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:18,764 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:50:18,780 DEBUG [Thread-1861 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,788 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,790 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56276, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,791 DEBUG [Thread-1867 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,791 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:50:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,798 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56288, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,803 DEBUG [Thread-1874 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:18,804 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:50:18,805 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:56302, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:18,807 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2] 2024-11-15T00:50:18,841 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/f95463774b3d4bdd8cc41357bbb51333 is 50, key is test_row_0/A:col10/1731631818753/Put/seqid=0 2024-11-15T00:50:18,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:50:18,892 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742236_1412 (size=16681) 2024-11-15T00:50:18,900 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:18,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631878867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:18,901 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:18,905 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:50:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:18,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:18,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:18,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:18,904 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:18,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631878894, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:18,908 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:18,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:18,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631878899, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:18,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631878904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:18,930 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:18,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631878905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:18,930 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:18,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,014 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631879007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,015 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631879021, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,047 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631879042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,048 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631879043, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,048 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,054 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631879044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,056 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,061 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:50:19,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,065 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:50:19,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:50:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:19,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,220 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631879218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,235 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,247 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631879242, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,248 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,263 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631879260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,263 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,264 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631879260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,274 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631879269, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,276 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,301 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/f95463774b3d4bdd8cc41357bbb51333 2024-11-15T00:50:19,377 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:50:19,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:19,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:50:19,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,419 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/ac23a2b888fa44209b5bb25f7b7924ea is 50, key is test_row_0/B:col10/1731631818753/Put/seqid=0 2024-11-15T00:50:19,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742237_1413 (size=12001) 2024-11-15T00:50:19,445 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/ac23a2b888fa44209b5bb25f7b7924ea 2024-11-15T00:50:19,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:50:19,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:19,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,537 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,555 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/a029144b1fbd46159672b3983242cbd8 is 50, key is test_row_0/C:col10/1731631818753/Put/seqid=0 2024-11-15T00:50:19,563 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631879546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631879557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,565 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,568 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631879568, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,578 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,583 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742238_1414 (size=12001) 2024-11-15T00:50:19,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/a029144b1fbd46159672b3983242cbd8 2024-11-15T00:50:19,586 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631879581, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,587 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,597 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:19,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631879592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:19,600 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:19,628 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/f95463774b3d4bdd8cc41357bbb51333 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/f95463774b3d4bdd8cc41357bbb51333 2024-11-15T00:50:19,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/f95463774b3d4bdd8cc41357bbb51333, entries=250, sequenceid=13, filesize=16.3 K 2024-11-15T00:50:19,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/ac23a2b888fa44209b5bb25f7b7924ea as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ac23a2b888fa44209b5bb25f7b7924ea 2024-11-15T00:50:19,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ac23a2b888fa44209b5bb25f7b7924ea, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T00:50:19,664 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/a029144b1fbd46159672b3983242cbd8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a029144b1fbd46159672b3983242cbd8 2024-11-15T00:50:19,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:50:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,698 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] handler.RSProcedureHandler(58): pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=120 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a029144b1fbd46159672b3983242cbd8, entries=150, sequenceid=13, filesize=11.7 K 2024-11-15T00:50:19,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=147.60 KB/151140 for 0015d049952a9227751d617c5d05c387 in 936ms, sequenceid=13, compaction requested=false 2024-11-15T00:50:19,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=120 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:19,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:19,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=120 2024-11-15T00:50:19,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:19,856 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:50:19,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/a80f13c573244ed0ad91da6b7ee1e984 is 50, key is test_row_0/A:col10/1731631818897/Put/seqid=0 2024-11-15T00:50:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:50:19,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742239_1415 (size=12001) 2024-11-15T00:50:19,919 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/a80f13c573244ed0ad91da6b7ee1e984 2024-11-15T00:50:19,941 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/28bc70ca1e1d4e33ad6c0962ceaa1aa9 is 50, key is test_row_0/B:col10/1731631818897/Put/seqid=0 2024-11-15T00:50:19,971 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742240_1416 (size=12001) 2024-11-15T00:50:19,971 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/28bc70ca1e1d4e33ad6c0962ceaa1aa9 2024-11-15T00:50:19,985 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/f57ac424d0c64ef4ac32227b282fc88a is 50, key is test_row_0/C:col10/1731631818897/Put/seqid=0 2024-11-15T00:50:20,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742241_1417 (size=12001) 2024-11-15T00:50:20,085 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:20,085 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:20,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631880101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,115 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631880108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631880110, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631880114, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,123 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,124 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,126 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,126 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631880118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,127 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631880217, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,223 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,231 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631880227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,232 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631880237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631880240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,243 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,416 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=38 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/f57ac424d0c64ef4ac32227b282fc88a 2024-11-15T00:50:20,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/a80f13c573244ed0ad91da6b7ee1e984 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a80f13c573244ed0ad91da6b7ee1e984 2024-11-15T00:50:20,426 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a80f13c573244ed0ad91da6b7ee1e984, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T00:50:20,434 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631880427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,435 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/28bc70ca1e1d4e33ad6c0962ceaa1aa9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28bc70ca1e1d4e33ad6c0962ceaa1aa9 2024-11-15T00:50:20,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631880436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,440 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,441 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28bc70ca1e1d4e33ad6c0962ceaa1aa9, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T00:50:20,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/f57ac424d0c64ef4ac32227b282fc88a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/f57ac424d0c64ef4ac32227b282fc88a 2024-11-15T00:50:20,453 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/f57ac424d0c64ef4ac32227b282fc88a, entries=150, sequenceid=38, filesize=11.7 K 2024-11-15T00:50:20,461 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=67.09 KB/68700 for 0015d049952a9227751d617c5d05c387 in 605ms, sequenceid=38, compaction requested=false 2024-11-15T00:50:20,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:20,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=120}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=120 2024-11-15T00:50:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=120 2024-11-15T00:50:20,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-15T00:50:20,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7190 sec 2024-11-15T00:50:20,477 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=119, table=TestAcidGuarantees in 1.7380 sec 2024-11-15T00:50:20,478 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:20,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-15T00:50:20,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:20,491 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/04485c2cb151428b8fd477145c09f2dd is 50, key is test_row_0/A:col10/1731631820110/Put/seqid=0 2024-11-15T00:50:20,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742242_1418 (size=16681) 2024-11-15T00:50:20,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631880587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631880589, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,605 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,720 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631880710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,721 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,721 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631880710, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,724 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,725 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,762 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631880748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631880748, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,764 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,768 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=119 2024-11-15T00:50:20,907 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:20,907 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:20,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/04485c2cb151428b8fd477145c09f2dd 2024-11-15T00:50:20,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-15T00:50:20,910 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:20,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:50:20,912 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:20,912 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:20,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631880927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,937 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:20,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631880929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,939 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:20,940 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:20,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/d41afd629f624a068907e943af9fab91 is 50, key is test_row_0/B:col10/1731631820110/Put/seqid=0 2024-11-15T00:50:20,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742243_1419 (size=12001) 2024-11-15T00:50:21,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:50:21,067 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T00:50:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:21,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,067 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:21,068 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:21,069 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:21,146 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:21,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631881137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:21,146 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2242 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:21,147 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:21,220 WARN [regionserver/fb2f6cb7846a:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 6, running: 0 2024-11-15T00:50:21,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T00:50:21,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:21,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:21,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:21,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:21,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:50:21,253 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:21,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631881247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:21,253 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:21,261 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:21,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631881247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:21,264 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:21,280 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:21,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631881272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:21,281 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:21,290 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:21,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631881277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:21,291 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:21,380 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T00:50:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,381 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:21,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:21,381 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/d41afd629f624a068907e943af9fab91 2024-11-15T00:50:21,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:21,402 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/0ae9c3e5d1e1495882132a0b4f3ba9eb is 50, key is test_row_0/C:col10/1731631820110/Put/seqid=0 2024-11-15T00:50:21,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742244_1420 (size=12001) 2024-11-15T00:50:21,442 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=53 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/0ae9c3e5d1e1495882132a0b4f3ba9eb 2024-11-15T00:50:21,448 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/04485c2cb151428b8fd477145c09f2dd as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/04485c2cb151428b8fd477145c09f2dd 2024-11-15T00:50:21,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/04485c2cb151428b8fd477145c09f2dd, entries=250, sequenceid=53, filesize=16.3 K 2024-11-15T00:50:21,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/d41afd629f624a068907e943af9fab91 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/d41afd629f624a068907e943af9fab91 2024-11-15T00:50:21,463 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/d41afd629f624a068907e943af9fab91, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T00:50:21,464 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/0ae9c3e5d1e1495882132a0b4f3ba9eb as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0ae9c3e5d1e1495882132a0b4f3ba9eb 2024-11-15T00:50:21,471 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0ae9c3e5d1e1495882132a0b4f3ba9eb, entries=150, sequenceid=53, filesize=11.7 K 2024-11-15T00:50:21,476 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 0015d049952a9227751d617c5d05c387 in 998ms, sequenceid=53, compaction requested=true 2024-11-15T00:50:21,476 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:21,476 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:21,477 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45363 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:21,477 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:21,477 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,478 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/f95463774b3d4bdd8cc41357bbb51333, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a80f13c573244ed0ad91da6b7ee1e984, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/04485c2cb151428b8fd477145c09f2dd] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=44.3 K 2024-11-15T00:50:21,478 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f95463774b3d4bdd8cc41357bbb51333, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731631818753 2024-11-15T00:50:21,484 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a80f13c573244ed0ad91da6b7ee1e984, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731631818860 2024-11-15T00:50:21,490 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04485c2cb151428b8fd477145c09f2dd, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631820110 2024-11-15T00:50:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:21,493 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:21,493 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:21,494 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:21,494 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:21,494 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,494 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ac23a2b888fa44209b5bb25f7b7924ea, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28bc70ca1e1d4e33ad6c0962ceaa1aa9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/d41afd629f624a068907e943af9fab91] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=35.2 K 2024-11-15T00:50:21,495 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ac23a2b888fa44209b5bb25f7b7924ea, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731631818753 2024-11-15T00:50:21,495 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 28bc70ca1e1d4e33ad6c0962ceaa1aa9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731631818860 2024-11-15T00:50:21,496 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d41afd629f624a068907e943af9fab91, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631820110 2024-11-15T00:50:21,500 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#343 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:21,501 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/b25937f9946e4593a9ebe83f2739e6a9 is 50, key is test_row_0/A:col10/1731631820110/Put/seqid=0 2024-11-15T00:50:21,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742245_1421 (size=12104) 2024-11-15T00:50:21,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:50:21,537 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/b25937f9946e4593a9ebe83f2739e6a9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b25937f9946e4593a9ebe83f2739e6a9 2024-11-15T00:50:21,543 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#344 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:21,543 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/1538bc22619f41e2b24c9685d0524be6 is 50, key is test_row_0/B:col10/1731631820110/Put/seqid=0 2024-11-15T00:50:21,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-15T00:50:21,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-15T00:50:21,562 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into b25937f9946e4593a9ebe83f2739e6a9(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:21,562 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:21,562 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=13, startTime=1731631821476; duration=0sec 2024-11-15T00:50:21,562 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:21,562 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:21,562 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:21,565 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:21,565 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:21,565 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,565 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a029144b1fbd46159672b3983242cbd8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/f57ac424d0c64ef4ac32227b282fc88a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0ae9c3e5d1e1495882132a0b4f3ba9eb] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=35.2 K 2024-11-15T00:50:21,566 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a029144b1fbd46159672b3983242cbd8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1731631818753 2024-11-15T00:50:21,567 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f57ac424d0c64ef4ac32227b282fc88a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=38, earliestPutTs=1731631818860 2024-11-15T00:50:21,567 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ae9c3e5d1e1495882132a0b4f3ba9eb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631820110 2024-11-15T00:50:21,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/10b00a3b2d88416391d3b0946b045ef1 is 50, key is test_row_0/A:col10/1731631820585/Put/seqid=0 2024-11-15T00:50:21,610 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742246_1422 (size=12104) 2024-11-15T00:50:21,635 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#346 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:21,635 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/dfd4bb3053a046bfa87871a51176e011 is 50, key is test_row_0/C:col10/1731631820110/Put/seqid=0 2024-11-15T00:50:21,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742247_1423 (size=12001) 2024-11-15T00:50:21,680 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/10b00a3b2d88416391d3b0946b045ef1 2024-11-15T00:50:21,707 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742248_1424 (size=12104) 2024-11-15T00:50:21,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/bd4ce87da29042a981892ca764f01528 is 50, key is test_row_0/B:col10/1731631820585/Put/seqid=0 2024-11-15T00:50:21,722 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/dfd4bb3053a046bfa87871a51176e011 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/dfd4bb3053a046bfa87871a51176e011 2024-11-15T00:50:21,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742249_1425 (size=12001) 2024-11-15T00:50:21,738 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into dfd4bb3053a046bfa87871a51176e011(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:21,738 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:21,738 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=13, startTime=1731631821493; duration=0sec 2024-11-15T00:50:21,739 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:21,739 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:21,740 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/bd4ce87da29042a981892ca764f01528 2024-11-15T00:50:21,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:21,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:21,765 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:50:21,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/5339deaa331e4a36aa3f336223f9591b is 50, key is test_row_0/C:col10/1731631820585/Put/seqid=0 2024-11-15T00:50:21,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742250_1426 (size=12001) 2024-11-15T00:50:21,782 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/5339deaa331e4a36aa3f336223f9591b 2024-11-15T00:50:21,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/10b00a3b2d88416391d3b0946b045ef1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/10b00a3b2d88416391d3b0946b045ef1 2024-11-15T00:50:21,796 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/10b00a3b2d88416391d3b0946b045ef1, entries=150, sequenceid=74, filesize=11.7 K 2024-11-15T00:50:21,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/bd4ce87da29042a981892ca764f01528 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bd4ce87da29042a981892ca764f01528 2024-11-15T00:50:21,807 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bd4ce87da29042a981892ca764f01528, entries=150, sequenceid=74, filesize=11.7 K 2024-11-15T00:50:21,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/5339deaa331e4a36aa3f336223f9591b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5339deaa331e4a36aa3f336223f9591b 2024-11-15T00:50:21,816 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5339deaa331e4a36aa3f336223f9591b, entries=150, sequenceid=74, filesize=11.7 K 2024-11-15T00:50:21,817 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=33.54 KB/34350 for 0015d049952a9227751d617c5d05c387 in 271ms, sequenceid=74, compaction requested=false 2024-11-15T00:50:21,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:21,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:21,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-15T00:50:21,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-15T00:50:21,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-15T00:50:21,827 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 910 msec 2024-11-15T00:50:21,830 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 921 msec 2024-11-15T00:50:21,854 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:21,854 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:50:21,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:21,867 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/de7bc3b0da604da6a699e7cfa1a4d140 is 50, key is test_row_0/A:col10/1731631821854/Put/seqid=0 2024-11-15T00:50:21,906 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742251_1427 (size=16681) 2024-11-15T00:50:21,912 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/de7bc3b0da604da6a699e7cfa1a4d140 2024-11-15T00:50:21,945 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/6fcba9767ae14d3b95553c6ac789c3e9 is 50, key is test_row_0/B:col10/1731631821854/Put/seqid=0 2024-11-15T00:50:21,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742252_1428 (size=12001) 2024-11-15T00:50:22,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/6fcba9767ae14d3b95553c6ac789c3e9 2024-11-15T00:50:22,026 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/a277d3bfcd9a4845bc337dac76a420c8 is 50, key is test_row_0/C:col10/1731631821854/Put/seqid=0 2024-11-15T00:50:22,029 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/1538bc22619f41e2b24c9685d0524be6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/1538bc22619f41e2b24c9685d0524be6 2024-11-15T00:50:22,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-15T00:50:22,047 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:22,048 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-15T00:50:22,050 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:22,052 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:22,052 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:22,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:50:22,059 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into 1538bc22619f41e2b24c9685d0524be6(size=11.8 K), total size for store is 23.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:22,060 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:22,060 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=13, startTime=1731631821493; duration=0sec 2024-11-15T00:50:22,060 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:22,060 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:22,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742253_1429 (size=12001) 2024-11-15T00:50:22,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631882085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631882087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,111 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,112 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:50:22,216 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T00:50:22,216 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:22,217 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:22,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:22,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:22,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631882218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,237 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,237 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631882218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,240 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,309 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631882304, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,313 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:22,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,314 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,331 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631882312, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,332 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2231 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,333 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:50:22,379 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T00:50:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:22,380 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:22,380 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:22,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:22,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631882447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,456 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631882448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,458 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,460 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,486 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=88 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/a277d3bfcd9a4845bc337dac76a420c8 2024-11-15T00:50:22,503 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/de7bc3b0da604da6a699e7cfa1a4d140 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/de7bc3b0da604da6a699e7cfa1a4d140 2024-11-15T00:50:22,508 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/de7bc3b0da604da6a699e7cfa1a4d140, entries=250, sequenceid=88, filesize=16.3 K 2024-11-15T00:50:22,513 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/6fcba9767ae14d3b95553c6ac789c3e9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/6fcba9767ae14d3b95553c6ac789c3e9 2024-11-15T00:50:22,517 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/6fcba9767ae14d3b95553c6ac789c3e9, entries=150, sequenceid=88, filesize=11.7 K 2024-11-15T00:50:22,519 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/a277d3bfcd9a4845bc337dac76a420c8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a277d3bfcd9a4845bc337dac76a420c8 2024-11-15T00:50:22,524 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a277d3bfcd9a4845bc337dac76a420c8, entries=150, sequenceid=88, filesize=11.7 K 2024-11-15T00:50:22,525 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0015d049952a9227751d617c5d05c387 in 671ms, sequenceid=88, compaction requested=true 2024-11-15T00:50:22,525 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:22,526 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:22,527 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:22,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:22,528 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:22,528 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:22,528 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:22,528 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:22,528 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b25937f9946e4593a9ebe83f2739e6a9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/10b00a3b2d88416391d3b0946b045ef1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/de7bc3b0da604da6a699e7cfa1a4d140] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=39.8 K 2024-11-15T00:50:22,528 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:22,529 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:22,534 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b25937f9946e4593a9ebe83f2739e6a9, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631820110 2024-11-15T00:50:22,536 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 10b00a3b2d88416391d3b0946b045ef1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731631820573 2024-11-15T00:50:22,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-15T00:50:22,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:22,540 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:22,540 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:22,540 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T00:50:22,540 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:22,540 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/1538bc22619f41e2b24c9685d0524be6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bd4ce87da29042a981892ca764f01528, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/6fcba9767ae14d3b95553c6ac789c3e9] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=35.3 K 2024-11-15T00:50:22,540 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting de7bc3b0da604da6a699e7cfa1a4d140, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731631821787 2024-11-15T00:50:22,543 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1538bc22619f41e2b24c9685d0524be6, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631820110 2024-11-15T00:50:22,544 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bd4ce87da29042a981892ca764f01528, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731631820573 2024-11-15T00:50:22,544 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fcba9767ae14d3b95553c6ac789c3e9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731631821824 2024-11-15T00:50:22,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/b8745c01fd974e139ddd30e16163eab1 is 50, key is test_row_0/A:col10/1731631822046/Put/seqid=0 2024-11-15T00:50:22,564 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#353 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:22,565 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/663ada3f688446eb92422b0691730886 is 50, key is test_row_0/B:col10/1731631821854/Put/seqid=0 2024-11-15T00:50:22,582 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#354 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:22,583 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/9405c8433512455cadc5dfe00056c449 is 50, key is test_row_0/A:col10/1731631821854/Put/seqid=0 2024-11-15T00:50:22,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742254_1430 (size=12001) 2024-11-15T00:50:22,637 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/b8745c01fd974e139ddd30e16163eab1 2024-11-15T00:50:22,650 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742255_1431 (size=12207) 2024-11-15T00:50:22,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742256_1432 (size=12207) 2024-11-15T00:50:22,666 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/b7b6362b8ae5483fb5cdc82186305718 is 50, key is test_row_0/B:col10/1731631822046/Put/seqid=0 2024-11-15T00:50:22,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:50:22,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742257_1433 (size=12001) 2024-11-15T00:50:22,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:22,775 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:22,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631882841, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631882843, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,862 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631882967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631882967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:22,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:22,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:23,080 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/663ada3f688446eb92422b0691730886 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/663ada3f688446eb92422b0691730886 2024-11-15T00:50:23,080 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/9405c8433512455cadc5dfe00056c449 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9405c8433512455cadc5dfe00056c449 2024-11-15T00:50:23,086 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into 9405c8433512455cadc5dfe00056c449(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:23,086 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:23,086 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=13, startTime=1731631822525; duration=0sec 2024-11-15T00:50:23,086 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:23,086 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:23,086 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:23,087 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:23,087 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:23,087 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:23,087 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/dfd4bb3053a046bfa87871a51176e011, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5339deaa331e4a36aa3f336223f9591b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a277d3bfcd9a4845bc337dac76a420c8] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=35.3 K 2024-11-15T00:50:23,088 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting dfd4bb3053a046bfa87871a51176e011, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=53, earliestPutTs=1731631820110 2024-11-15T00:50:23,088 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5339deaa331e4a36aa3f336223f9591b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1731631820573 2024-11-15T00:50:23,089 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a277d3bfcd9a4845bc337dac76a420c8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731631821824 2024-11-15T00:50:23,101 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#356 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:23,102 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/51b76cd0bacf4746ba6c4f3665fee2a9 is 50, key is test_row_0/C:col10/1731631821854/Put/seqid=0 2024-11-15T00:50:23,111 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into 663ada3f688446eb92422b0691730886(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:23,111 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:23,111 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=13, startTime=1731631822528; duration=0sec 2024-11-15T00:50:23,111 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:23,111 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:23,144 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/b7b6362b8ae5483fb5cdc82186305718 2024-11-15T00:50:23,150 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742258_1434 (size=12207) 2024-11-15T00:50:23,179 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:23,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631883169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:23,179 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/b506a35151624aa28b9e16a197114619 is 50, key is test_row_0/C:col10/1731631822046/Put/seqid=0 2024-11-15T00:50:23,180 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4276 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:23,181 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:23,182 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/51b76cd0bacf4746ba6c4f3665fee2a9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/51b76cd0bacf4746ba6c4f3665fee2a9 2024-11-15T00:50:23,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:50:23,193 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into 51b76cd0bacf4746ba6c4f3665fee2a9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:23,193 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:23,193 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=13, startTime=1731631822529; duration=0sec 2024-11-15T00:50:23,193 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:23,193 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:23,204 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:23,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631883189, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:23,205 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:23,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:23,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631883196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:23,211 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:23,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742259_1435 (size=12001) 2024-11-15T00:50:23,525 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631883517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:23,525 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:23,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631883517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:23,526 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:23,619 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=113 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/b506a35151624aa28b9e16a197114619 2024-11-15T00:50:23,626 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/b8745c01fd974e139ddd30e16163eab1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b8745c01fd974e139ddd30e16163eab1 2024-11-15T00:50:23,633 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b8745c01fd974e139ddd30e16163eab1, entries=150, sequenceid=113, filesize=11.7 K 2024-11-15T00:50:23,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/b7b6362b8ae5483fb5cdc82186305718 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/b7b6362b8ae5483fb5cdc82186305718 2024-11-15T00:50:23,639 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/b7b6362b8ae5483fb5cdc82186305718, entries=150, sequenceid=113, filesize=11.7 K 2024-11-15T00:50:23,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/b506a35151624aa28b9e16a197114619 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/b506a35151624aa28b9e16a197114619 2024-11-15T00:50:23,647 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/b506a35151624aa28b9e16a197114619, entries=150, sequenceid=113, filesize=11.7 K 2024-11-15T00:50:23,652 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0015d049952a9227751d617c5d05c387 in 1111ms, sequenceid=113, compaction requested=false 2024-11-15T00:50:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:23,652 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-15T00:50:23,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-15T00:50:23,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-15T00:50:23,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6010 sec 2024-11-15T00:50:23,657 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.6080 sec 2024-11-15T00:50:24,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:24,040 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:24,040 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T00:50:24,056 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/746b4672f0e8411bba395919d377d528 is 50, key is test_row_0/A:col10/1731631822839/Put/seqid=0 2024-11-15T00:50:24,060 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742260_1436 (size=14391) 2024-11-15T00:50:24,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:24,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631884180, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,194 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631884181, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,194 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:24,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-15T00:50:24,197 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:24,197 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:24,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-15T00:50:24,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:50:24,200 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:24,200 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:24,201 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:24,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:24,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631884297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:24,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631884297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,308 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:24,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:50:24,335 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:24,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631884327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:24,335 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4223 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:24,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,336 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:24,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T00:50:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:24,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:24,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:24,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631884357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:24,365 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,366 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:24,476 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/746b4672f0e8411bba395919d377d528 2024-11-15T00:50:24,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/15cfec22d638447f99a098ba7b8ac866 is 50, key is test_row_0/B:col10/1731631822839/Put/seqid=0 2024-11-15T00:50:24,511 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T00:50:24,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:24,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:24,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:24,512 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,512 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:50:24,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631884518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,524 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:24,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631884518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,524 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:24,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742261_1437 (size=12051) 2024-11-15T00:50:24,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T00:50:24,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:24,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:24,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:24,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T00:50:24,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:24,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:24,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:24,822 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:50:24,836 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:24,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631884829, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,837 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:24,852 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:24,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631884840, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:24,856 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:24,953 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/15cfec22d638447f99a098ba7b8ac866 2024-11-15T00:50:24,978 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/9379bba82b5c47d6bb22989e5f548054 is 50, key is test_row_0/C:col10/1731631822839/Put/seqid=0 2024-11-15T00:50:24,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T00:50:24,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:24,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:24,981 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:24,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:25,013 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742262_1438 (size=12051) 2024-11-15T00:50:25,140 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T00:50:25,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:25,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:25,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:25,140 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:25,140 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:25,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:25,293 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T00:50:25,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:25,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:25,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:25,294 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:25,294 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:25,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:25,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:50:25,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:25,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631885348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:25,360 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:25,371 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:25,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631885366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:25,372 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:25,415 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/9379bba82b5c47d6bb22989e5f548054 2024-11-15T00:50:25,426 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/746b4672f0e8411bba395919d377d528 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/746b4672f0e8411bba395919d377d528 2024-11-15T00:50:25,435 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/746b4672f0e8411bba395919d377d528, entries=200, sequenceid=129, filesize=14.1 K 2024-11-15T00:50:25,436 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/15cfec22d638447f99a098ba7b8ac866 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/15cfec22d638447f99a098ba7b8ac866 2024-11-15T00:50:25,441 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/15cfec22d638447f99a098ba7b8ac866, entries=150, sequenceid=129, filesize=11.8 K 2024-11-15T00:50:25,442 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/9379bba82b5c47d6bb22989e5f548054 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9379bba82b5c47d6bb22989e5f548054 2024-11-15T00:50:25,447 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T00:50:25,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:25,448 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:25,448 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:25,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:25,454 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9379bba82b5c47d6bb22989e5f548054, entries=150, sequenceid=129, filesize=11.8 K 2024-11-15T00:50:25,455 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 0015d049952a9227751d617c5d05c387 in 1415ms, sequenceid=129, compaction requested=true 2024-11-15T00:50:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:25,455 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:25,455 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:50:25,455 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:25,456 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38599 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:25,456 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:25,456 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:25,456 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:25,456 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:25,456 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:25,456 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/663ada3f688446eb92422b0691730886, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/b7b6362b8ae5483fb5cdc82186305718, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/15cfec22d638447f99a098ba7b8ac866] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=35.4 K 2024-11-15T00:50:25,456 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9405c8433512455cadc5dfe00056c449, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b8745c01fd974e139ddd30e16163eab1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/746b4672f0e8411bba395919d377d528] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=37.7 K 2024-11-15T00:50:25,457 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9405c8433512455cadc5dfe00056c449, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731631821824 2024-11-15T00:50:25,457 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 663ada3f688446eb92422b0691730886, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731631821824 2024-11-15T00:50:25,457 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b8745c01fd974e139ddd30e16163eab1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731631822046 2024-11-15T00:50:25,457 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b7b6362b8ae5483fb5cdc82186305718, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731631822046 2024-11-15T00:50:25,457 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 746b4672f0e8411bba395919d377d528, keycount=200, bloomtype=ROW, size=14.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631822801 2024-11-15T00:50:25,457 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 15cfec22d638447f99a098ba7b8ac866, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631822801 2024-11-15T00:50:25,476 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#361 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:25,477 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/55a6931c3a3d475186976e27f7aa150f is 50, key is test_row_0/B:col10/1731631822839/Put/seqid=0 2024-11-15T00:50:25,484 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#362 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:25,485 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/03dd4ef125a04bcfac6c5ab4e1d3da1f is 50, key is test_row_0/A:col10/1731631822839/Put/seqid=0 2024-11-15T00:50:25,540 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742264_1440 (size=12359) 2024-11-15T00:50:25,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742263_1439 (size=12359) 2024-11-15T00:50:25,568 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/55a6931c3a3d475186976e27f7aa150f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/55a6931c3a3d475186976e27f7aa150f 2024-11-15T00:50:25,586 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into 55a6931c3a3d475186976e27f7aa150f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:25,586 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:25,586 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=13, startTime=1731631825455; duration=0sec 2024-11-15T00:50:25,586 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:25,587 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:25,587 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:25,593 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36259 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:25,593 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:25,593 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:25,593 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/51b76cd0bacf4746ba6c4f3665fee2a9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/b506a35151624aa28b9e16a197114619, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9379bba82b5c47d6bb22989e5f548054] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=35.4 K 2024-11-15T00:50:25,596 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 51b76cd0bacf4746ba6c4f3665fee2a9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=88, earliestPutTs=1731631821824 2024-11-15T00:50:25,600 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b506a35151624aa28b9e16a197114619, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=113, earliestPutTs=1731631822046 2024-11-15T00:50:25,601 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9379bba82b5c47d6bb22989e5f548054, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631822801 2024-11-15T00:50:25,601 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-15T00:50:25,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:25,602 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T00:50:25,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/5cad0eafc3604417983ff825280569e2 is 50, key is test_row_0/A:col10/1731631824179/Put/seqid=0 2024-11-15T00:50:25,619 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#364 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:25,620 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/a5ca082436ce4bfd8785ee3c578d6c53 is 50, key is test_row_0/C:col10/1731631822839/Put/seqid=0 2024-11-15T00:50:25,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742265_1441 (size=12151) 2024-11-15T00:50:25,633 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742266_1442 (size=12359) 2024-11-15T00:50:25,646 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/a5ca082436ce4bfd8785ee3c578d6c53 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a5ca082436ce4bfd8785ee3c578d6c53 2024-11-15T00:50:25,665 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into a5ca082436ce4bfd8785ee3c578d6c53(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:25,665 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:25,665 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=13, startTime=1731631825455; duration=0sec 2024-11-15T00:50:25,665 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:25,665 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:25,953 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/03dd4ef125a04bcfac6c5ab4e1d3da1f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/03dd4ef125a04bcfac6c5ab4e1d3da1f 2024-11-15T00:50:25,959 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into 03dd4ef125a04bcfac6c5ab4e1d3da1f(size=12.1 K), total size for store is 12.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:25,959 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:25,959 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=13, startTime=1731631825455; duration=0sec 2024-11-15T00:50:25,959 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:25,959 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:26,048 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/5cad0eafc3604417983ff825280569e2 2024-11-15T00:50:26,091 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/50f0dd3f18344056a78f9445e42c40df is 50, key is test_row_0/B:col10/1731631824179/Put/seqid=0 2024-11-15T00:50:26,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742267_1443 (size=12151) 2024-11-15T00:50:26,149 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/50f0dd3f18344056a78f9445e42c40df 2024-11-15T00:50:26,162 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/5db27adb9d8b42b3a1670013c7edaad4 is 50, key is test_row_0/C:col10/1731631824179/Put/seqid=0 2024-11-15T00:50:26,198 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742268_1444 (size=12151) 2024-11-15T00:50:26,201 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/5db27adb9d8b42b3a1670013c7edaad4 2024-11-15T00:50:26,226 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/5cad0eafc3604417983ff825280569e2 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/5cad0eafc3604417983ff825280569e2 2024-11-15T00:50:26,246 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/5cad0eafc3604417983ff825280569e2, entries=150, sequenceid=153, filesize=11.9 K 2024-11-15T00:50:26,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/50f0dd3f18344056a78f9445e42c40df as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/50f0dd3f18344056a78f9445e42c40df 2024-11-15T00:50:26,265 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/50f0dd3f18344056a78f9445e42c40df, entries=150, sequenceid=153, filesize=11.9 K 2024-11-15T00:50:26,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/5db27adb9d8b42b3a1670013c7edaad4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5db27adb9d8b42b3a1670013c7edaad4 2024-11-15T00:50:26,271 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5db27adb9d8b42b3a1670013c7edaad4, entries=150, sequenceid=153, filesize=11.9 K 2024-11-15T00:50:26,272 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 0015d049952a9227751d617c5d05c387 in 670ms, sequenceid=153, compaction requested=false 2024-11-15T00:50:26,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:26,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:26,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-15T00:50:26,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-15T00:50:26,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-15T00:50:26,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0870 sec 2024-11-15T00:50:26,294 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.0940 sec 2024-11-15T00:50:26,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-15T00:50:26,347 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:26,348 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:26,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-15T00:50:26,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T00:50:26,354 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:26,355 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:26,356 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:26,427 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:26,427 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:50:26,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:26,442 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/4542c0ce6b3e4205978a373c019fda66 is 50, key is test_row_0/A:col10/1731631826402/Put/seqid=0 2024-11-15T00:50:26,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T00:50:26,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742269_1445 (size=12151) 2024-11-15T00:50:26,496 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/4542c0ce6b3e4205978a373c019fda66 2024-11-15T00:50:26,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T00:50:26,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:26,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:26,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:26,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,554 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/03790e061c8e4953847dcafa6fa068c4 is 50, key is test_row_0/B:col10/1731631826402/Put/seqid=0 2024-11-15T00:50:26,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742270_1446 (size=12151) 2024-11-15T00:50:26,605 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/03790e061c8e4953847dcafa6fa068c4 2024-11-15T00:50:26,650 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/6f4ab458376449f0a037254ef211c54e is 50, key is test_row_0/C:col10/1731631826402/Put/seqid=0 2024-11-15T00:50:26,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T00:50:26,680 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T00:50:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:26,681 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,681 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,688 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742271_1447 (size=12151) 2024-11-15T00:50:26,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:26,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631886717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:26,731 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:26,731 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:26,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631886720, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:26,736 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:26,836 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T00:50:26,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:26,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:26,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:26,837 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,837 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:26,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631886837, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:26,848 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:26,854 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:26,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631886847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:26,855 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:26,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T00:50:26,990 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T00:50:26,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:26,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:26,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:26,991 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] handler.RSProcedureHandler(58): pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,991 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=128 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:26,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=128 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:27,060 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:27,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631887057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,061 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:27,078 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:27,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631887067, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,078 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:27,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=166 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/6f4ab458376449f0a037254ef211c54e 2024-11-15T00:50:27,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/4542c0ce6b3e4205978a373c019fda66 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/4542c0ce6b3e4205978a373c019fda66 2024-11-15T00:50:27,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/4542c0ce6b3e4205978a373c019fda66, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T00:50:27,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/03790e061c8e4953847dcafa6fa068c4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/03790e061c8e4953847dcafa6fa068c4 2024-11-15T00:50:27,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/03790e061c8e4953847dcafa6fa068c4, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T00:50:27,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/6f4ab458376449f0a037254ef211c54e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6f4ab458376449f0a037254ef211c54e 2024-11-15T00:50:27,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6f4ab458376449f0a037254ef211c54e, entries=150, sequenceid=166, filesize=11.9 K 2024-11-15T00:50:27,121 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0015d049952a9227751d617c5d05c387 in 694ms, sequenceid=166, compaction requested=true 2024-11-15T00:50:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:27,121 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:27,121 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:27,121 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:27,122 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:27,122 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:27,122 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:27,122 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/55a6931c3a3d475186976e27f7aa150f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/50f0dd3f18344056a78f9445e42c40df, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/03790e061c8e4953847dcafa6fa068c4] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=35.8 K 2024-11-15T00:50:27,122 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:27,122 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:27,122 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:27,122 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 55a6931c3a3d475186976e27f7aa150f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631822801 2024-11-15T00:50:27,122 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/03dd4ef125a04bcfac6c5ab4e1d3da1f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/5cad0eafc3604417983ff825280569e2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/4542c0ce6b3e4205978a373c019fda66] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=35.8 K 2024-11-15T00:50:27,123 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 50f0dd3f18344056a78f9445e42c40df, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731631824159 2024-11-15T00:50:27,123 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 03dd4ef125a04bcfac6c5ab4e1d3da1f, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631822801 2024-11-15T00:50:27,123 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5cad0eafc3604417983ff825280569e2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731631824159 2024-11-15T00:50:27,123 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 03790e061c8e4953847dcafa6fa068c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631826402 2024-11-15T00:50:27,123 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4542c0ce6b3e4205978a373c019fda66, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631826402 2024-11-15T00:50:27,146 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#370 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:27,146 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/c8e3ba7e6f9a46f9867f59474c0038bd is 50, key is test_row_0/A:col10/1731631826402/Put/seqid=0 2024-11-15T00:50:27,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-15T00:50:27,154 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#371 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:27,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:27,155 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/8c6a911a7aca46d3b69ac1c019ba1243 is 50, key is test_row_0/B:col10/1731631826402/Put/seqid=0 2024-11-15T00:50:27,155 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:50:27,186 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/a8ae1bc687fd429a8a80c00036d3b234 is 50, key is test_row_0/A:col10/1731631826666/Put/seqid=0 2024-11-15T00:50:27,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:27,203 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:27,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742273_1449 (size=12561) 2024-11-15T00:50:27,212 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742272_1448 (size=12561) 2024-11-15T00:50:27,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742274_1450 (size=12151) 2024-11-15T00:50:27,227 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/c8e3ba7e6f9a46f9867f59474c0038bd as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c8e3ba7e6f9a46f9867f59474c0038bd 2024-11-15T00:50:27,233 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into c8e3ba7e6f9a46f9867f59474c0038bd(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:27,233 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:27,233 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=13, startTime=1731631827121; duration=0sec 2024-11-15T00:50:27,233 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:27,233 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:27,233 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:27,235 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36661 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:27,235 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:27,235 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:27,235 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a5ca082436ce4bfd8785ee3c578d6c53, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5db27adb9d8b42b3a1670013c7edaad4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6f4ab458376449f0a037254ef211c54e] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=35.8 K 2024-11-15T00:50:27,236 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a5ca082436ce4bfd8785ee3c578d6c53, keycount=150, bloomtype=ROW, size=12.1 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631822801 2024-11-15T00:50:27,237 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5db27adb9d8b42b3a1670013c7edaad4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731631824159 2024-11-15T00:50:27,237 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f4ab458376449f0a037254ef211c54e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631826402 2024-11-15T00:50:27,246 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#373 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:27,246 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/0578644c246e4ab0991b6c8af96a9c1e is 50, key is test_row_0/C:col10/1731631826402/Put/seqid=0 2024-11-15T00:50:27,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742275_1451 (size=12561) 2024-11-15T00:50:27,256 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/0578644c246e4ab0991b6c8af96a9c1e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0578644c246e4ab0991b6c8af96a9c1e 2024-11-15T00:50:27,264 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into 0578644c246e4ab0991b6c8af96a9c1e(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:27,264 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:27,264 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=13, startTime=1731631827121; duration=0sec 2024-11-15T00:50:27,264 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:27,264 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:27,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:27,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631887315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:27,371 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:27,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631887366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,377 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:27,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:27,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631887387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,393 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:27,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:27,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631887427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:27,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T00:50:27,629 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/a8ae1bc687fd429a8a80c00036d3b234 2024-11-15T00:50:27,638 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/8c6a911a7aca46d3b69ac1c019ba1243 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8c6a911a7aca46d3b69ac1c019ba1243 2024-11-15T00:50:27,643 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into 8c6a911a7aca46d3b69ac1c019ba1243(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:27,643 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:27,643 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=13, startTime=1731631827121; duration=0sec 2024-11-15T00:50:27,643 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:27,643 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:27,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/012234564ea24e6aafbaaadf36f61e3c is 50, key is test_row_0/B:col10/1731631826666/Put/seqid=0 2024-11-15T00:50:27,652 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:27,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631887648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,656 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:27,680 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742276_1452 (size=12151) 2024-11-15T00:50:27,900 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:27,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631887888, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:27,902 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,903 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:27,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:27,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631887913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,922 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:27,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:27,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631887972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:27,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:28,079 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/012234564ea24e6aafbaaadf36f61e3c 2024-11-15T00:50:28,092 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/5a5f9f0efee742c4a4d245412722ae35 is 50, key is test_row_0/C:col10/1731631826666/Put/seqid=0 2024-11-15T00:50:28,105 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742277_1453 (size=12151) 2024-11-15T00:50:28,196 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-15T00:50:28,196 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-15T00:50:28,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:28,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631888347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:28,351 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8238 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:28,352 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:28,385 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:28,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631888378, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:28,387 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8285 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:28,387 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:28,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:28,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631888486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:28,491 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:28,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:28,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T00:50:28,508 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=191 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/5a5f9f0efee742c4a4d245412722ae35 2024-11-15T00:50:28,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/a8ae1bc687fd429a8a80c00036d3b234 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a8ae1bc687fd429a8a80c00036d3b234 2024-11-15T00:50:28,520 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a8ae1bc687fd429a8a80c00036d3b234, entries=150, sequenceid=191, filesize=11.9 K 2024-11-15T00:50:28,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/012234564ea24e6aafbaaadf36f61e3c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/012234564ea24e6aafbaaadf36f61e3c 2024-11-15T00:50:28,525 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/012234564ea24e6aafbaaadf36f61e3c, entries=150, sequenceid=191, filesize=11.9 K 2024-11-15T00:50:28,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/5a5f9f0efee742c4a4d245412722ae35 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5a5f9f0efee742c4a4d245412722ae35 2024-11-15T00:50:28,537 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5a5f9f0efee742c4a4d245412722ae35, entries=150, sequenceid=191, filesize=11.9 K 2024-11-15T00:50:28,538 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0015d049952a9227751d617c5d05c387 in 1383ms, sequenceid=191, compaction requested=false 2024-11-15T00:50:28,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:28,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:28,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-15T00:50:28,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-15T00:50:28,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-15T00:50:28,540 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1830 sec 2024-11-15T00:50:28,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 2.1940 sec 2024-11-15T00:50:28,912 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:28,912 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T00:50:28,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:28,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/cf1ca71f97cf4e138185ede93ad32d70 is 50, key is test_row_0/A:col10/1731631827221/Put/seqid=0 2024-11-15T00:50:28,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742278_1454 (size=14541) 2024-11-15T00:50:29,066 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:29,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631889057, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,068 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:29,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:29,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631889069, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,082 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:29,179 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:29,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631889177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,180 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:29,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:29,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631889187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,195 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:29,333 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/cf1ca71f97cf4e138185ede93ad32d70 2024-11-15T00:50:29,345 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/abc6f9065a45404586cf6c67554ae3f9 is 50, key is test_row_0/B:col10/1731631827221/Put/seqid=0 2024-11-15T00:50:29,375 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742279_1455 (size=12151) 2024-11-15T00:50:29,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:29,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631889387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,398 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:29,424 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:29,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631889407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:29,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,424 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:29,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:29,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631889506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:29,515 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,516 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:29,710 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:29,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631889706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,710 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:29,744 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:29,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631889736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:29,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:29,744 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:29,775 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/abc6f9065a45404586cf6c67554ae3f9 2024-11-15T00:50:29,780 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/ecaaec0477324aea9c08d1c5ca988ff2 is 50, key is test_row_0/C:col10/1731631827221/Put/seqid=0 2024-11-15T00:50:29,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742280_1456 (size=12151) 2024-11-15T00:50:30,184 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/ecaaec0477324aea9c08d1c5ca988ff2 2024-11-15T00:50:30,195 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/cf1ca71f97cf4e138185ede93ad32d70 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/cf1ca71f97cf4e138185ede93ad32d70 2024-11-15T00:50:30,199 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/cf1ca71f97cf4e138185ede93ad32d70, entries=200, sequenceid=206, filesize=14.2 K 2024-11-15T00:50:30,199 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/abc6f9065a45404586cf6c67554ae3f9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/abc6f9065a45404586cf6c67554ae3f9 2024-11-15T00:50:30,209 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/abc6f9065a45404586cf6c67554ae3f9, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T00:50:30,209 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/ecaaec0477324aea9c08d1c5ca988ff2 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/ecaaec0477324aea9c08d1c5ca988ff2 2024-11-15T00:50:30,213 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/ecaaec0477324aea9c08d1c5ca988ff2, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T00:50:30,214 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0015d049952a9227751d617c5d05c387 in 1302ms, sequenceid=206, compaction requested=true 2024-11-15T00:50:30,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:30,214 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:30,214 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:30,218 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:30,218 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39253 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:30,218 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:30,218 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:30,219 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c8e3ba7e6f9a46f9867f59474c0038bd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a8ae1bc687fd429a8a80c00036d3b234, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/cf1ca71f97cf4e138185ede93ad32d70] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=38.3 K 2024-11-15T00:50:30,219 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c8e3ba7e6f9a46f9867f59474c0038bd, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631826402 2024-11-15T00:50:30,218 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:30,220 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8ae1bc687fd429a8a80c00036d3b234, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731631826648 2024-11-15T00:50:30,220 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:30,220 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:30,220 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:30,220 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8c6a911a7aca46d3b69ac1c019ba1243, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/012234564ea24e6aafbaaadf36f61e3c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/abc6f9065a45404586cf6c67554ae3f9] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=36.0 K 2024-11-15T00:50:30,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:30,220 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting cf1ca71f97cf4e138185ede93ad32d70, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631827221 2024-11-15T00:50:30,221 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8c6a911a7aca46d3b69ac1c019ba1243, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631826402 2024-11-15T00:50:30,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:30,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:30,221 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:30,221 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:30,221 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:50:30,221 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 012234564ea24e6aafbaaadf36f61e3c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731631826648 2024-11-15T00:50:30,222 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting abc6f9065a45404586cf6c67554ae3f9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631827221 2024-11-15T00:50:30,228 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/817e1ea9198549b3836f84cc7416ef78 is 50, key is test_row_0/A:col10/1731631829040/Put/seqid=0 2024-11-15T00:50:30,243 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:30,243 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/f5a8a71d4fd541db8bbf61e65d812925 is 50, key is test_row_0/B:col10/1731631827221/Put/seqid=0 2024-11-15T00:50:30,243 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:30,244 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/67e77bf512b64e91b2194782e37c923c is 50, key is test_row_0/A:col10/1731631827221/Put/seqid=0 2024-11-15T00:50:30,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742281_1457 (size=14541) 2024-11-15T00:50:30,249 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/817e1ea9198549b3836f84cc7416ef78 2024-11-15T00:50:30,250 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742282_1458 (size=12663) 2024-11-15T00:50:30,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742283_1459 (size=12663) 2024-11-15T00:50:30,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/28901ceeb2d24ce69b9fce84f4e0de71 is 50, key is test_row_0/B:col10/1731631829040/Put/seqid=0 2024-11-15T00:50:30,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742284_1460 (size=12151) 2024-11-15T00:50:30,298 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:30,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631890285, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:30,298 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,299 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:30,302 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:30,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631890290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:30,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,303 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:30,412 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:30,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631890407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:30,413 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631890407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,413 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:30,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-15T00:50:30,517 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:30,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:30,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-15T00:50:30,527 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:30,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T00:50:30,528 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:30,529 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:30,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631890617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:30,619 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631890617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,619 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:30,620 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:30,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T00:50:30,655 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/f5a8a71d4fd541db8bbf61e65d812925 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f5a8a71d4fd541db8bbf61e65d812925 2024-11-15T00:50:30,664 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/67e77bf512b64e91b2194782e37c923c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/67e77bf512b64e91b2194782e37c923c 2024-11-15T00:50:30,669 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into 67e77bf512b64e91b2194782e37c923c(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:30,669 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into f5a8a71d4fd541db8bbf61e65d812925(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:30,669 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:30,669 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:30,669 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=13, startTime=1731631830218; duration=0sec 2024-11-15T00:50:30,669 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=13, startTime=1731631830214; duration=0sec 2024-11-15T00:50:30,669 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:30,670 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:30,670 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:30,670 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:30,670 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:30,672 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:30,672 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:30,672 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:30,673 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0578644c246e4ab0991b6c8af96a9c1e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5a5f9f0efee742c4a4d245412722ae35, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/ecaaec0477324aea9c08d1c5ca988ff2] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=36.0 K 2024-11-15T00:50:30,673 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0578644c246e4ab0991b6c8af96a9c1e, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=166, earliestPutTs=1731631826402 2024-11-15T00:50:30,673 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5a5f9f0efee742c4a4d245412722ae35, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=191, earliestPutTs=1731631826648 2024-11-15T00:50:30,673 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ecaaec0477324aea9c08d1c5ca988ff2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631827221 2024-11-15T00:50:30,682 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#383 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:30,682 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/6263179dd09947bfbd77d3ceb9de65fe is 50, key is test_row_0/C:col10/1731631827221/Put/seqid=0 2024-11-15T00:50:30,682 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T00:50:30,682 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:30,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:30,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:30,683 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:30,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:30,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:30,685 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/28901ceeb2d24ce69b9fce84f4e0de71 2024-11-15T00:50:30,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742285_1461 (size=12663) 2024-11-15T00:50:30,704 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/071ea17f6d5143e1a8ff470fe0908e4d is 50, key is test_row_0/C:col10/1731631829040/Put/seqid=0 2024-11-15T00:50:30,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742286_1462 (size=12151) 2024-11-15T00:50:30,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=231 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/071ea17f6d5143e1a8ff470fe0908e4d 2024-11-15T00:50:30,766 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/817e1ea9198549b3836f84cc7416ef78 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/817e1ea9198549b3836f84cc7416ef78 2024-11-15T00:50:30,786 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/817e1ea9198549b3836f84cc7416ef78, entries=200, sequenceid=231, filesize=14.2 K 2024-11-15T00:50:30,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/28901ceeb2d24ce69b9fce84f4e0de71 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28901ceeb2d24ce69b9fce84f4e0de71 2024-11-15T00:50:30,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28901ceeb2d24ce69b9fce84f4e0de71, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T00:50:30,810 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/071ea17f6d5143e1a8ff470fe0908e4d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/071ea17f6d5143e1a8ff470fe0908e4d 2024-11-15T00:50:30,818 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/071ea17f6d5143e1a8ff470fe0908e4d, entries=150, sequenceid=231, filesize=11.9 K 2024-11-15T00:50:30,818 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0015d049952a9227751d617c5d05c387 in 597ms, sequenceid=231, compaction requested=false 2024-11-15T00:50:30,818 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:30,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-15T00:50:30,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:30,843 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:50:30,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T00:50:30,847 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/c7d0c48efec540ba8b4dcb7750bacc4a is 50, key is test_row_0/A:col10/1731631830288/Put/seqid=0 2024-11-15T00:50:30,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742287_1463 (size=12151) 2024-11-15T00:50:30,866 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/c7d0c48efec540ba8b4dcb7750bacc4a 2024-11-15T00:50:30,882 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/f4a96be1a2894c99a6925a985eb34b47 is 50, key is test_row_0/B:col10/1731631830288/Put/seqid=0 2024-11-15T00:50:30,891 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742288_1464 (size=12151) 2024-11-15T00:50:30,893 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/f4a96be1a2894c99a6925a985eb34b47 2024-11-15T00:50:30,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/16d4b4fcf3cc4249b2d983ff618440b0 is 50, key is test_row_0/C:col10/1731631830288/Put/seqid=0 2024-11-15T00:50:30,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742289_1465 (size=12151) 2024-11-15T00:50:30,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:30,938 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:31,052 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631891048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:31,058 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,058 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631891048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,058 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:31,116 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/6263179dd09947bfbd77d3ceb9de65fe as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6263179dd09947bfbd77d3ceb9de65fe 2024-11-15T00:50:31,138 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into 6263179dd09947bfbd77d3ceb9de65fe(size=12.4 K), total size for store is 24.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:31,138 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:31,138 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=13, startTime=1731631830221; duration=0sec 2024-11-15T00:50:31,138 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:31,138 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:31,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T00:50:31,163 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,163 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631891156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,163 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:31,174 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631891167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,175 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:31,341 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=244 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/16d4b4fcf3cc4249b2d983ff618440b0 2024-11-15T00:50:31,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631891367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,380 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:31,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/c7d0c48efec540ba8b4dcb7750bacc4a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c7d0c48efec540ba8b4dcb7750bacc4a 2024-11-15T00:50:31,399 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631891392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,404 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:31,423 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c7d0c48efec540ba8b4dcb7750bacc4a, entries=150, sequenceid=244, filesize=11.9 K 2024-11-15T00:50:31,432 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/f4a96be1a2894c99a6925a985eb34b47 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f4a96be1a2894c99a6925a985eb34b47 2024-11-15T00:50:31,473 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f4a96be1a2894c99a6925a985eb34b47, entries=150, sequenceid=244, filesize=11.9 K 2024-11-15T00:50:31,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/16d4b4fcf3cc4249b2d983ff618440b0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/16d4b4fcf3cc4249b2d983ff618440b0 2024-11-15T00:50:31,492 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/16d4b4fcf3cc4249b2d983ff618440b0, entries=150, sequenceid=244, filesize=11.9 K 2024-11-15T00:50:31,498 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 0015d049952a9227751d617c5d05c387 in 655ms, sequenceid=244, compaction requested=true 2024-11-15T00:50:31,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:31,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:31,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-15T00:50:31,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-15T00:50:31,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-15T00:50:31,519 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 986 msec 2024-11-15T00:50:31,522 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 1.0020 sec 2024-11-15T00:50:31,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:31,553 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:31,553 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-15T00:50:31,568 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/3aec4f3239db4eeeb0e156ec9e0ade43 is 50, key is test_row_0/A:col10/1731631831046/Put/seqid=0 2024-11-15T00:50:31,608 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742290_1466 (size=14741) 2024-11-15T00:50:31,611 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/3aec4f3239db4eeeb0e156ec9e0ade43 2024-11-15T00:50:31,643 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/bdbf164fc8bf43e6baa95020220b01ee is 50, key is test_row_0/B:col10/1731631831046/Put/seqid=0 2024-11-15T00:50:31,659 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631891652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,664 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:31,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-15T00:50:31,667 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:31,668 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:31,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-15T00:50:31,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T00:50:31,678 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:31,679 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:31,679 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:31,682 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742291_1467 (size=12301) 2024-11-15T00:50:31,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631891688, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,700 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:31,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631891717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,727 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:31,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631891767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,776 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:31,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T00:50:31,842 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T00:50:31,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:31,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:31,842 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:31,842 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:31,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:31,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:31,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:31,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631891987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:31,999 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:32,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T00:50:32,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T00:50:32,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:32,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,088 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/bdbf164fc8bf43e6baa95020220b01ee 2024-11-15T00:50:32,132 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/4c15a5af5b704b69aa0d8f8751a833da is 50, key is test_row_0/C:col10/1731631831046/Put/seqid=0 2024-11-15T00:50:32,171 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T00:50:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,172 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742292_1468 (size=12301) 2024-11-15T00:50:32,218 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:32,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631892208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:32,220 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:32,246 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:32,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631892239, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:32,247 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:32,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T00:50:32,320 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:32,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631892313, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:32,320 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:32,321 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:32,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T00:50:32,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,499 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T00:50:32,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,500 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:32,585 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=271 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/4c15a5af5b704b69aa0d8f8751a833da 2024-11-15T00:50:32,605 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/3aec4f3239db4eeeb0e156ec9e0ade43 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/3aec4f3239db4eeeb0e156ec9e0ade43 2024-11-15T00:50:32,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/3aec4f3239db4eeeb0e156ec9e0ade43, entries=200, sequenceid=271, filesize=14.4 K 2024-11-15T00:50:32,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/bdbf164fc8bf43e6baa95020220b01ee as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bdbf164fc8bf43e6baa95020220b01ee 2024-11-15T00:50:32,623 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bdbf164fc8bf43e6baa95020220b01ee, entries=150, sequenceid=271, filesize=12.0 K 2024-11-15T00:50:32,625 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/4c15a5af5b704b69aa0d8f8751a833da as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/4c15a5af5b704b69aa0d8f8751a833da 2024-11-15T00:50:32,630 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/4c15a5af5b704b69aa0d8f8751a833da, entries=150, sequenceid=271, filesize=12.0 K 2024-11-15T00:50:32,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0015d049952a9227751d617c5d05c387 in 1077ms, sequenceid=271, compaction requested=true 2024-11-15T00:50:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:50:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:32,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T00:50:32,631 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:32,631 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:32,633 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 54096 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:32,633 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:32,633 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,633 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:32,633 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:32,633 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/67e77bf512b64e91b2194782e37c923c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/817e1ea9198549b3836f84cc7416ef78, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c7d0c48efec540ba8b4dcb7750bacc4a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/3aec4f3239db4eeeb0e156ec9e0ade43] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=52.8 K 2024-11-15T00:50:32,633 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,634 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6263179dd09947bfbd77d3ceb9de65fe, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/071ea17f6d5143e1a8ff470fe0908e4d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/16d4b4fcf3cc4249b2d983ff618440b0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/4c15a5af5b704b69aa0d8f8751a833da] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=48.1 K 2024-11-15T00:50:32,636 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 67e77bf512b64e91b2194782e37c923c, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631827221 2024-11-15T00:50:32,636 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6263179dd09947bfbd77d3ceb9de65fe, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631827221 2024-11-15T00:50:32,637 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 071ea17f6d5143e1a8ff470fe0908e4d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731631829003 2024-11-15T00:50:32,637 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 817e1ea9198549b3836f84cc7416ef78, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731631829003 2024-11-15T00:50:32,637 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 16d4b4fcf3cc4249b2d983ff618440b0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731631830283 2024-11-15T00:50:32,637 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c7d0c48efec540ba8b4dcb7750bacc4a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731631830283 2024-11-15T00:50:32,638 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aec4f3239db4eeeb0e156ec9e0ade43, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731631831036 2024-11-15T00:50:32,638 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4c15a5af5b704b69aa0d8f8751a833da, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731631831036 2024-11-15T00:50:32,648 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:32,648 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#392 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:32,648 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/9170d529bb7d4d9b9455ef2a45d42b53 is 50, key is test_row_0/A:col10/1731631831046/Put/seqid=0 2024-11-15T00:50:32,650 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/43f5f1ae6cd94dc090e43b10b12e2ce6 is 50, key is test_row_0/C:col10/1731631831046/Put/seqid=0 2024-11-15T00:50:32,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-15T00:50:32,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:32,657 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T00:50:32,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742293_1469 (size=12949) 2024-11-15T00:50:32,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742294_1470 (size=12949) 2024-11-15T00:50:32,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/2cd31791f5494ef5a2e45b341068db39 is 50, key is test_row_0/A:col10/1731631831633/Put/seqid=0 2024-11-15T00:50:32,737 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742295_1471 (size=12301) 2024-11-15T00:50:32,741 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/2cd31791f5494ef5a2e45b341068db39 2024-11-15T00:50:32,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/f91463ef257849e796fdb100014598e9 is 50, key is test_row_0/B:col10/1731631831633/Put/seqid=0 2024-11-15T00:50:32,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742296_1472 (size=12301) 2024-11-15T00:50:32,815 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/f91463ef257849e796fdb100014598e9 2024-11-15T00:50:32,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T00:50:32,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/92592c4b04bd48f08089e7416fa5b23d is 50, key is test_row_0/C:col10/1731631831633/Put/seqid=0 2024-11-15T00:50:32,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:32,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:32,897 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742297_1473 (size=12301) 2024-11-15T00:50:33,116 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/9170d529bb7d4d9b9455ef2a45d42b53 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9170d529bb7d4d9b9455ef2a45d42b53 2024-11-15T00:50:33,117 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/43f5f1ae6cd94dc090e43b10b12e2ce6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/43f5f1ae6cd94dc090e43b10b12e2ce6 2024-11-15T00:50:33,124 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into 43f5f1ae6cd94dc090e43b10b12e2ce6(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:33,125 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:33,125 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=12, startTime=1731631832631; duration=0sec 2024-11-15T00:50:33,125 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:33,125 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:33,125 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:50:33,129 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into 9170d529bb7d4d9b9455ef2a45d42b53(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:33,129 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:33,129 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=12, startTime=1731631832631; duration=0sec 2024-11-15T00:50:33,129 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:33,129 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:33,130 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49266 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:50:33,130 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:33,130 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:33,130 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f5a8a71d4fd541db8bbf61e65d812925, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28901ceeb2d24ce69b9fce84f4e0de71, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f4a96be1a2894c99a6925a985eb34b47, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bdbf164fc8bf43e6baa95020220b01ee] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=48.1 K 2024-11-15T00:50:33,130 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f5a8a71d4fd541db8bbf61e65d812925, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631827221 2024-11-15T00:50:33,130 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28901ceeb2d24ce69b9fce84f4e0de71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=231, earliestPutTs=1731631829003 2024-11-15T00:50:33,131 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f4a96be1a2894c99a6925a985eb34b47, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=244, earliestPutTs=1731631830283 2024-11-15T00:50:33,131 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bdbf164fc8bf43e6baa95020220b01ee, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731631831036 2024-11-15T00:50:33,149 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#396 average throughput is 0.73 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:33,150 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/c5f52b46bbcf44caab11cae788329d9e is 50, key is test_row_0/B:col10/1731631831046/Put/seqid=0 2024-11-15T00:50:33,197 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742298_1474 (size=12949) 2024-11-15T00:50:33,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631893237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:33,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:33,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631893237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:33,248 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,249 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:33,265 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:33,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631893260, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:33,266 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_0' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2218 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,267 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:33,308 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=281 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/92592c4b04bd48f08089e7416fa5b23d 2024-11-15T00:50:33,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/2cd31791f5494ef5a2e45b341068db39 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2cd31791f5494ef5a2e45b341068db39 2024-11-15T00:50:33,351 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2cd31791f5494ef5a2e45b341068db39, entries=150, sequenceid=281, filesize=12.0 K 2024-11-15T00:50:33,352 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/f91463ef257849e796fdb100014598e9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f91463ef257849e796fdb100014598e9 2024-11-15T00:50:33,356 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f91463ef257849e796fdb100014598e9, entries=150, sequenceid=281, filesize=12.0 K 2024-11-15T00:50:33,356 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:33,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631893347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,357 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:33,359 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/92592c4b04bd48f08089e7416fa5b23d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/92592c4b04bd48f08089e7416fa5b23d 2024-11-15T00:50:33,365 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/92592c4b04bd48f08089e7416fa5b23d, entries=150, sequenceid=281, filesize=12.0 K 2024-11-15T00:50:33,365 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 0015d049952a9227751d617c5d05c387 in 708ms, sequenceid=281, compaction requested=false 2024-11-15T00:50:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:33,366 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-15T00:50:33,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-15T00:50:33,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-15T00:50:33,369 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6890 sec 2024-11-15T00:50:33,371 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.7010 sec 2024-11-15T00:50:33,570 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:33,570 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T00:50:33,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:33,587 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/110bdcfb112e4f2ea17274f008066e94 is 50, key is test_row_0/A:col10/1731631833220/Put/seqid=0 2024-11-15T00:50:33,626 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:33,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631893612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:33,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,627 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:33,629 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/c5f52b46bbcf44caab11cae788329d9e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c5f52b46bbcf44caab11cae788329d9e 2024-11-15T00:50:33,631 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742299_1475 (size=14741) 2024-11-15T00:50:33,632 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/110bdcfb112e4f2ea17274f008066e94 2024-11-15T00:50:33,656 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into c5f52b46bbcf44caab11cae788329d9e(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:33,656 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:33,656 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=12, startTime=1731631832631; duration=0sec 2024-11-15T00:50:33,656 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:33,656 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:33,662 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/91525edb582441d28bffdad55458ab5b is 50, key is test_row_0/B:col10/1731631833220/Put/seqid=0 2024-11-15T00:50:33,691 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742300_1476 (size=12301) 2024-11-15T00:50:33,697 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/91525edb582441d28bffdad55458ab5b 2024-11-15T00:50:33,705 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/417a0e3fd0c242a28cb9ee34b7f79c19 is 50, key is test_row_0/C:col10/1731631833220/Put/seqid=0 2024-11-15T00:50:33,733 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742301_1477 (size=12301) 2024-11-15T00:50:33,737 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/417a0e3fd0c242a28cb9ee34b7f79c19 2024-11-15T00:50:33,740 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/110bdcfb112e4f2ea17274f008066e94 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/110bdcfb112e4f2ea17274f008066e94 2024-11-15T00:50:33,746 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:33,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631893737, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:33,747 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:33,750 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/110bdcfb112e4f2ea17274f008066e94, entries=200, sequenceid=310, filesize=14.4 K 2024-11-15T00:50:33,751 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/91525edb582441d28bffdad55458ab5b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/91525edb582441d28bffdad55458ab5b 2024-11-15T00:50:33,759 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/91525edb582441d28bffdad55458ab5b, entries=150, sequenceid=310, filesize=12.0 K 2024-11-15T00:50:33,760 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/417a0e3fd0c242a28cb9ee34b7f79c19 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/417a0e3fd0c242a28cb9ee34b7f79c19 2024-11-15T00:50:33,764 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/417a0e3fd0c242a28cb9ee34b7f79c19, entries=150, sequenceid=310, filesize=12.0 K 2024-11-15T00:50:33,765 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 0015d049952a9227751d617c5d05c387 in 195ms, sequenceid=310, compaction requested=true 2024-11-15T00:50:33,765 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:33,765 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:33,766 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:33,766 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:33,766 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:33,766 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:33,766 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:33,766 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9170d529bb7d4d9b9455ef2a45d42b53, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2cd31791f5494ef5a2e45b341068db39, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/110bdcfb112e4f2ea17274f008066e94] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=39.1 K 2024-11-15T00:50:33,766 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:33,766 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:33,767 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:33,767 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c5f52b46bbcf44caab11cae788329d9e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f91463ef257849e796fdb100014598e9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/91525edb582441d28bffdad55458ab5b] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=36.7 K 2024-11-15T00:50:33,767 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9170d529bb7d4d9b9455ef2a45d42b53, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731631831036 2024-11-15T00:50:33,767 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f52b46bbcf44caab11cae788329d9e, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731631831036 2024-11-15T00:50:33,767 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2cd31791f5494ef5a2e45b341068db39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731631831624 2024-11-15T00:50:33,767 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f91463ef257849e796fdb100014598e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731631831624 2024-11-15T00:50:33,767 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 110bdcfb112e4f2ea17274f008066e94, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731631833172 2024-11-15T00:50:33,767 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 91525edb582441d28bffdad55458ab5b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731631833172 2024-11-15T00:50:33,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:33,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:33,776 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:33,780 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#400 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:33,780 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/2815edce48dc4365bec31e17f5ca42d0 is 50, key is test_row_0/B:col10/1731631833220/Put/seqid=0 2024-11-15T00:50:33,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742302_1478 (size=13051) 2024-11-15T00:50:33,789 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/2815edce48dc4365bec31e17f5ca42d0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/2815edce48dc4365bec31e17f5ca42d0 2024-11-15T00:50:33,794 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#401 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:33,794 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into 2815edce48dc4365bec31e17f5ca42d0(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:33,795 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:33,795 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=13, startTime=1731631833766; duration=0sec 2024-11-15T00:50:33,795 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:33,795 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:33,795 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/159a5784dcb240bd8628ee2c499e0b02 is 50, key is test_row_0/A:col10/1731631833220/Put/seqid=0 2024-11-15T00:50:33,795 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:33,798 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:33,798 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:33,798 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:33,798 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/43f5f1ae6cd94dc090e43b10b12e2ce6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/92592c4b04bd48f08089e7416fa5b23d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/417a0e3fd0c242a28cb9ee34b7f79c19] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=36.7 K 2024-11-15T00:50:33,798 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 43f5f1ae6cd94dc090e43b10b12e2ce6, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=271, earliestPutTs=1731631831036 2024-11-15T00:50:33,799 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 92592c4b04bd48f08089e7416fa5b23d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=281, earliestPutTs=1731631831624 2024-11-15T00:50:33,799 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 417a0e3fd0c242a28cb9ee34b7f79c19, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731631833172 2024-11-15T00:50:33,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-15T00:50:33,832 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:33,836 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:33,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-15T00:50:33,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T00:50:33,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742303_1479 (size=13051) 2024-11-15T00:50:33,840 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:33,841 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:33,842 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/7bf1d554db584539a1d7a54eac935f54 is 50, key is test_row_0/C:col10/1731631833220/Put/seqid=0 2024-11-15T00:50:33,842 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:33,842 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:33,852 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/159a5784dcb240bd8628ee2c499e0b02 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/159a5784dcb240bd8628ee2c499e0b02 2024-11-15T00:50:33,864 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into 159a5784dcb240bd8628ee2c499e0b02(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:33,864 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:33,864 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=13, startTime=1731631833765; duration=0sec 2024-11-15T00:50:33,864 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:33,864 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:33,878 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742304_1480 (size=13051) 2024-11-15T00:50:33,892 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/7bf1d554db584539a1d7a54eac935f54 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7bf1d554db584539a1d7a54eac935f54 2024-11-15T00:50:33,900 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into 7bf1d554db584539a1d7a54eac935f54(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:33,900 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:33,900 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=13, startTime=1731631833776; duration=0sec 2024-11-15T00:50:33,900 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:33,900 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:33,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T00:50:33,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:33,974 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:33,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:50:33,988 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/c69e685c5710479f9965530a224525ff is 50, key is test_row_0/A:col10/1731631833967/Put/seqid=0 2024-11-15T00:50:33,995 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742305_1481 (size=14741) 2024-11-15T00:50:33,997 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T00:50:33,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:33,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:33,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:33,998 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:33,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:33,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:33,999 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/c69e685c5710479f9965530a224525ff 2024-11-15T00:50:34,058 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/19eec3197d5d4be1921f8a68796c1cab is 50, key is test_row_0/B:col10/1731631833967/Put/seqid=0 2024-11-15T00:50:34,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742306_1482 (size=12301) 2024-11-15T00:50:34,140 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/19eec3197d5d4be1921f8a68796c1cab 2024-11-15T00:50:34,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T00:50:34,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:34,155 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,155 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,157 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T00:50:34,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,194 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/2ecab4a33181418399c00999da4d6c30 is 50, key is test_row_0/C:col10/1731631833967/Put/seqid=0 2024-11-15T00:50:34,220 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742307_1483 (size=12301) 2024-11-15T00:50:34,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=325 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/2ecab4a33181418399c00999da4d6c30 2024-11-15T00:50:34,260 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/c69e685c5710479f9965530a224525ff as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c69e685c5710479f9965530a224525ff 2024-11-15T00:50:34,264 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c69e685c5710479f9965530a224525ff, entries=200, sequenceid=325, filesize=14.4 K 2024-11-15T00:50:34,273 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/19eec3197d5d4be1921f8a68796c1cab as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/19eec3197d5d4be1921f8a68796c1cab 2024-11-15T00:50:34,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/19eec3197d5d4be1921f8a68796c1cab, entries=150, sequenceid=325, filesize=12.0 K 2024-11-15T00:50:34,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/2ecab4a33181418399c00999da4d6c30 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/2ecab4a33181418399c00999da4d6c30 2024-11-15T00:50:34,292 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/2ecab4a33181418399c00999da4d6c30, entries=150, sequenceid=325, filesize=12.0 K 2024-11-15T00:50:34,293 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=134.18 KB/137400 for 0015d049952a9227751d617c5d05c387 in 319ms, sequenceid=325, compaction requested=false 2024-11-15T00:50:34,293 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:34,303 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:34,303 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T00:50:34,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:34,312 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/2b2a3d8a5bb549fab222a4ec35f1b6f8 is 50, key is test_row_0/A:col10/1731631834301/Put/seqid=0 2024-11-15T00:50:34,325 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T00:50:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,364 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742308_1484 (size=12301) 2024-11-15T00:50:34,372 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/2b2a3d8a5bb549fab222a4ec35f1b6f8 2024-11-15T00:50:34,421 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/c68275fef6ef4676947e73928f5dc5b3 is 50, key is test_row_0/B:col10/1731631834301/Put/seqid=0 2024-11-15T00:50:34,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:34,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631894452, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:34,467 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:34,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T00:50:34,469 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742309_1485 (size=12301) 2024-11-15T00:50:34,477 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/c68275fef6ef4676947e73928f5dc5b3 2024-11-15T00:50:34,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T00:50:34,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:34,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,482 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/29360e38c24f4de98ab4c212c0172e18 is 50, key is test_row_0/C:col10/1731631834301/Put/seqid=0 2024-11-15T00:50:34,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742310_1486 (size=12301) 2024-11-15T00:50:34,599 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:34,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631894588, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:34,602 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:34,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T00:50:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T00:50:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,822 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:34,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631894808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:34,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:34,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:34,823 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:34,938 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/29360e38c24f4de98ab4c212c0172e18 2024-11-15T00:50:34,943 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/2b2a3d8a5bb549fab222a4ec35f1b6f8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2b2a3d8a5bb549fab222a4ec35f1b6f8 2024-11-15T00:50:34,950 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2b2a3d8a5bb549fab222a4ec35f1b6f8, entries=150, sequenceid=349, filesize=12.0 K 2024-11-15T00:50:34,952 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/c68275fef6ef4676947e73928f5dc5b3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c68275fef6ef4676947e73928f5dc5b3 2024-11-15T00:50:34,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c68275fef6ef4676947e73928f5dc5b3, entries=150, sequenceid=349, filesize=12.0 K 2024-11-15T00:50:34,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/29360e38c24f4de98ab4c212c0172e18 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/29360e38c24f4de98ab4c212c0172e18 2024-11-15T00:50:34,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T00:50:34,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:34,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,971 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:34,973 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/29360e38c24f4de98ab4c212c0172e18, entries=150, sequenceid=349, filesize=12.0 K 2024-11-15T00:50:34,974 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 0015d049952a9227751d617c5d05c387 in 671ms, sequenceid=349, compaction requested=true 2024-11-15T00:50:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:34,974 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:34,974 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:34,974 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:34,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:34,976 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:34,976 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40093 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:34,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T00:50:34,977 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:34,977 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,977 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/159a5784dcb240bd8628ee2c499e0b02, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c69e685c5710479f9965530a224525ff, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2b2a3d8a5bb549fab222a4ec35f1b6f8] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=39.2 K 2024-11-15T00:50:34,977 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:34,977 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:34,977 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:34,977 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/2815edce48dc4365bec31e17f5ca42d0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/19eec3197d5d4be1921f8a68796c1cab, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c68275fef6ef4676947e73928f5dc5b3] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=36.8 K 2024-11-15T00:50:34,978 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2815edce48dc4365bec31e17f5ca42d0, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731631833172 2024-11-15T00:50:34,978 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 159a5784dcb240bd8628ee2c499e0b02, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731631833172 2024-11-15T00:50:34,978 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 19eec3197d5d4be1921f8a68796c1cab, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631833592 2024-11-15T00:50:34,978 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c69e685c5710479f9965530a224525ff, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631833592 2024-11-15T00:50:34,978 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c68275fef6ef4676947e73928f5dc5b3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731631834203 2024-11-15T00:50:34,978 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2b2a3d8a5bb549fab222a4ec35f1b6f8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731631834203 2024-11-15T00:50:35,004 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#409 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:35,004 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/37fa7a0f0c6545d9a27835a7ef6ef460 is 50, key is test_row_0/A:col10/1731631834301/Put/seqid=0 2024-11-15T00:50:35,012 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#410 average throughput is 0.94 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:35,013 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/8acb22a8de064603898724c0ec6ef1ec is 50, key is test_row_0/B:col10/1731631834301/Put/seqid=0 2024-11-15T00:50:35,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742312_1488 (size=13153) 2024-11-15T00:50:35,039 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/8acb22a8de064603898724c0ec6ef1ec as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8acb22a8de064603898724c0ec6ef1ec 2024-11-15T00:50:35,047 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into 8acb22a8de064603898724c0ec6ef1ec(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:35,047 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:35,047 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=13, startTime=1731631834974; duration=0sec 2024-11-15T00:50:35,047 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:35,047 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:35,047 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:35,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742311_1487 (size=13153) 2024-11-15T00:50:35,054 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37653 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:35,055 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:35,055 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:35,055 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7bf1d554db584539a1d7a54eac935f54, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/2ecab4a33181418399c00999da4d6c30, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/29360e38c24f4de98ab4c212c0172e18] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=36.8 K 2024-11-15T00:50:35,056 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf1d554db584539a1d7a54eac935f54, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731631833172 2024-11-15T00:50:35,056 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ecab4a33181418399c00999da4d6c30, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=325, earliestPutTs=1731631833592 2024-11-15T00:50:35,059 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 29360e38c24f4de98ab4c212c0172e18, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731631834203 2024-11-15T00:50:35,060 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/37fa7a0f0c6545d9a27835a7ef6ef460 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/37fa7a0f0c6545d9a27835a7ef6ef460 2024-11-15T00:50:35,068 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into 37fa7a0f0c6545d9a27835a7ef6ef460(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:35,068 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:35,068 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=13, startTime=1731631834974; duration=0sec 2024-11-15T00:50:35,068 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:35,068 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:35,080 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#411 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:35,081 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/534637522b974283a782a260cb35ed6f is 50, key is test_row_0/C:col10/1731631834301/Put/seqid=0 2024-11-15T00:50:35,100 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742313_1489 (size=13153) 2024-11-15T00:50:35,132 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/534637522b974283a782a260cb35ed6f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/534637522b974283a782a260cb35ed6f 2024-11-15T00:50:35,132 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-15T00:50:35,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:35,133 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T00:50:35,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:35,141 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:35,160 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/2ad7f4d32a6546d3b6ecba8950463127 is 50, key is test_row_0/A:col10/1731631834450/Put/seqid=0 2024-11-15T00:50:35,161 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into 534637522b974283a782a260cb35ed6f(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:35,161 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:35,161 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=13, startTime=1731631834974; duration=0sec 2024-11-15T00:50:35,161 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:35,161 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:35,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742314_1490 (size=12301) 2024-11-15T00:50:35,185 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/2ad7f4d32a6546d3b6ecba8950463127 2024-11-15T00:50:35,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/28a0da45e427454ca941f3b3c3f5d45a is 50, key is test_row_0/B:col10/1731631834450/Put/seqid=0 2024-11-15T00:50:35,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742315_1491 (size=12301) 2024-11-15T00:50:35,238 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/28a0da45e427454ca941f3b3c3f5d45a 2024-11-15T00:50:35,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/7eaa6fbb28e24c8e85737a58dfb45f5f is 50, key is test_row_0/C:col10/1731631834450/Put/seqid=0 2024-11-15T00:50:35,304 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742316_1492 (size=12301) 2024-11-15T00:50:35,307 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=363 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/7eaa6fbb28e24c8e85737a58dfb45f5f 2024-11-15T00:50:35,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/2ad7f4d32a6546d3b6ecba8950463127 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2ad7f4d32a6546d3b6ecba8950463127 2024-11-15T00:50:35,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631895319, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,335 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2ad7f4d32a6546d3b6ecba8950463127, entries=150, sequenceid=363, filesize=12.0 K 2024-11-15T00:50:35,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/28a0da45e427454ca941f3b3c3f5d45a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28a0da45e427454ca941f3b3c3f5d45a 2024-11-15T00:50:35,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631895325, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631895326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,338 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,342 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28a0da45e427454ca941f3b3c3f5d45a, entries=150, sequenceid=363, filesize=12.0 K 2024-11-15T00:50:35,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/7eaa6fbb28e24c8e85737a58dfb45f5f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7eaa6fbb28e24c8e85737a58dfb45f5f 2024-11-15T00:50:35,346 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7eaa6fbb28e24c8e85737a58dfb45f5f, entries=150, sequenceid=363, filesize=12.0 K 2024-11-15T00:50:35,347 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0015d049952a9227751d617c5d05c387 in 214ms, sequenceid=363, compaction requested=false 2024-11-15T00:50:35,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:35,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:35,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-15T00:50:35,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-15T00:50:35,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-15T00:50:35,351 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5070 sec 2024-11-15T00:50:35,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.5150 sec 2024-11-15T00:50:35,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:35,442 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:35,442 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:50:35,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/190ccb02eb584aa484ca84439554ddc2 is 50, key is test_row_0/A:col10/1731631835439/Put/seqid=0 2024-11-15T00:50:35,480 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631895466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,480 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,480 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631895467, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,481 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,482 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742317_1493 (size=14741) 2024-11-15T00:50:35,493 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631895481, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,493 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,494 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,597 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631895587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631895587, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,598 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,598 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631895597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,603 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631895807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,813 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,813 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631895807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:35,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631895812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:35,814 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:35,887 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/190ccb02eb584aa484ca84439554ddc2 2024-11-15T00:50:35,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/3eb9b9c18db0427791f675c512783814 is 50, key is test_row_0/B:col10/1731631835439/Put/seqid=0 2024-11-15T00:50:35,942 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742318_1494 (size=12301) 2024-11-15T00:50:35,942 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/3eb9b9c18db0427791f675c512783814 2024-11-15T00:50:35,954 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/df52c170996b475591ed6b7572dc972f is 50, key is test_row_0/C:col10/1731631835439/Put/seqid=0 2024-11-15T00:50:35,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742319_1495 (size=12301) 2024-11-15T00:50:35,961 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=389 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/df52c170996b475591ed6b7572dc972f 2024-11-15T00:50:35,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/190ccb02eb584aa484ca84439554ddc2 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/190ccb02eb584aa484ca84439554ddc2 2024-11-15T00:50:35,972 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/190ccb02eb584aa484ca84439554ddc2, entries=200, sequenceid=389, filesize=14.4 K 2024-11-15T00:50:35,973 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/3eb9b9c18db0427791f675c512783814 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/3eb9b9c18db0427791f675c512783814 2024-11-15T00:50:35,978 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/3eb9b9c18db0427791f675c512783814, entries=150, sequenceid=389, filesize=12.0 K 2024-11-15T00:50:35,979 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/df52c170996b475591ed6b7572dc972f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/df52c170996b475591ed6b7572dc972f 2024-11-15T00:50:35,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-15T00:50:35,987 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:35,987 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/df52c170996b475591ed6b7572dc972f, entries=150, sequenceid=389, filesize=12.0 K 2024-11-15T00:50:35,987 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:35,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-15T00:50:35,990 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:35,991 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 0015d049952a9227751d617c5d05c387 in 549ms, sequenceid=389, compaction requested=true 2024-11-15T00:50:35,991 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:35,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T00:50:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:35,992 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:35,992 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:35,992 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:35,992 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:35,992 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:35,994 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:35,994 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:35,994 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:35,994 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8acb22a8de064603898724c0ec6ef1ec, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28a0da45e427454ca941f3b3c3f5d45a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/3eb9b9c18db0427791f675c512783814] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=36.9 K 2024-11-15T00:50:35,994 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40195 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:35,994 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:35,994 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:35,995 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/37fa7a0f0c6545d9a27835a7ef6ef460, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2ad7f4d32a6546d3b6ecba8950463127, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/190ccb02eb584aa484ca84439554ddc2] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=39.3 K 2024-11-15T00:50:35,995 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8acb22a8de064603898724c0ec6ef1ec, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731631834203 2024-11-15T00:50:35,995 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 37fa7a0f0c6545d9a27835a7ef6ef460, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731631834203 2024-11-15T00:50:35,996 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ad7f4d32a6546d3b6ecba8950463127, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731631834311 2024-11-15T00:50:35,996 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 28a0da45e427454ca941f3b3c3f5d45a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731631834311 2024-11-15T00:50:35,996 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 190ccb02eb584aa484ca84439554ddc2, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731631835305 2024-11-15T00:50:35,996 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3eb9b9c18db0427791f675c512783814, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731631835306 2024-11-15T00:50:36,010 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:36,010 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#419 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:36,010 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/92b113c2cb6f4b439a89b19c738b6e45 is 50, key is test_row_0/B:col10/1731631835439/Put/seqid=0 2024-11-15T00:50:36,010 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/1ae8f47bea4441a0aa878d9b6e5665c0 is 50, key is test_row_0/A:col10/1731631835439/Put/seqid=0 2024-11-15T00:50:36,015 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742321_1497 (size=13255) 2024-11-15T00:50:36,019 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/1ae8f47bea4441a0aa878d9b6e5665c0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/1ae8f47bea4441a0aa878d9b6e5665c0 2024-11-15T00:50:36,025 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742320_1496 (size=13255) 2024-11-15T00:50:36,034 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into 1ae8f47bea4441a0aa878d9b6e5665c0(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:36,034 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:36,034 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=13, startTime=1731631835991; duration=0sec 2024-11-15T00:50:36,034 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:36,034 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:36,034 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:36,036 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/92b113c2cb6f4b439a89b19c738b6e45 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/92b113c2cb6f4b439a89b19c738b6e45 2024-11-15T00:50:36,036 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37755 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:36,036 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:36,036 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,036 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/534637522b974283a782a260cb35ed6f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7eaa6fbb28e24c8e85737a58dfb45f5f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/df52c170996b475591ed6b7572dc972f] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=36.9 K 2024-11-15T00:50:36,037 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 534637522b974283a782a260cb35ed6f, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1731631834203 2024-11-15T00:50:36,037 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7eaa6fbb28e24c8e85737a58dfb45f5f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=363, earliestPutTs=1731631834311 2024-11-15T00:50:36,038 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting df52c170996b475591ed6b7572dc972f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731631835306 2024-11-15T00:50:36,044 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into 92b113c2cb6f4b439a89b19c738b6e45(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:36,044 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:36,044 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=13, startTime=1731631835992; duration=0sec 2024-11-15T00:50:36,044 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:36,044 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:36,052 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#420 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:36,053 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/14715df5d07244388f8665e447a7758a is 50, key is test_row_0/C:col10/1731631835439/Put/seqid=0 2024-11-15T00:50:36,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742322_1498 (size=13255) 2024-11-15T00:50:36,084 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/14715df5d07244388f8665e447a7758a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/14715df5d07244388f8665e447a7758a 2024-11-15T00:50:36,091 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into 14715df5d07244388f8665e447a7758a(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:36,091 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:36,091 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=13, startTime=1731631835992; duration=0sec 2024-11-15T00:50:36,091 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:36,091 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:36,099 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T00:50:36,122 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:36,122 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T00:50:36,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:36,141 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/dd0109047f634bd4ba0b7988db24785b is 50, key is test_row_0/A:col10/1731631836120/Put/seqid=0 2024-11-15T00:50:36,153 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T00:50:36,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:36,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,154 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742323_1499 (size=14741) 2024-11-15T00:50:36,157 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/dd0109047f634bd4ba0b7988db24785b 2024-11-15T00:50:36,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/8e3e0398fbdb48fb9a872e7b77ff6d6a is 50, key is test_row_0/B:col10/1731631836120/Put/seqid=0 2024-11-15T00:50:36,182 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742324_1500 (size=12301) 2024-11-15T00:50:36,183 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/8e3e0398fbdb48fb9a872e7b77ff6d6a 2024-11-15T00:50:36,198 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/d7c8a71c92b546a6b03447da1bfbd35c is 50, key is test_row_0/C:col10/1731631836120/Put/seqid=0 2024-11-15T00:50:36,220 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631896210, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631896212, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,221 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,221 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 146 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631896213, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,222 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,258 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742325_1501 (size=12301) 2024-11-15T00:50:36,260 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=404 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/d7c8a71c92b546a6b03447da1bfbd35c 2024-11-15T00:50:36,266 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/dd0109047f634bd4ba0b7988db24785b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/dd0109047f634bd4ba0b7988db24785b 2024-11-15T00:50:36,272 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/dd0109047f634bd4ba0b7988db24785b, entries=200, sequenceid=404, filesize=14.4 K 2024-11-15T00:50:36,287 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/8e3e0398fbdb48fb9a872e7b77ff6d6a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8e3e0398fbdb48fb9a872e7b77ff6d6a 2024-11-15T00:50:36,297 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8e3e0398fbdb48fb9a872e7b77ff6d6a, entries=150, sequenceid=404, filesize=12.0 K 2024-11-15T00:50:36,297 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/d7c8a71c92b546a6b03447da1bfbd35c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d7c8a71c92b546a6b03447da1bfbd35c 2024-11-15T00:50:36,306 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T00:50:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T00:50:36,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,317 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d7c8a71c92b546a6b03447da1bfbd35c, entries=150, sequenceid=404, filesize=12.0 K 2024-11-15T00:50:36,323 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 0015d049952a9227751d617c5d05c387 in 201ms, sequenceid=404, compaction requested=false 2024-11-15T00:50:36,324 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:36,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:36,344 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:36,344 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:50:36,357 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/a0edb668681e4fceb94376781f6bc5cf is 50, key is test_row_0/A:col10/1731631836343/Put/seqid=0 2024-11-15T00:50:36,379 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631896377, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,379 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,380 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742326_1502 (size=17181) 2024-11-15T00:50:36,390 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631896379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631896379, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,390 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T00:50:36,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,491 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631896486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,492 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631896497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631896497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T00:50:36,629 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T00:50:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,629 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,630 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,630 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,708 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631896696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,709 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631896707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,713 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631896708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:36,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:36,790 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/a0edb668681e4fceb94376781f6bc5cf 2024-11-15T00:50:36,792 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T00:50:36,792 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,793 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,816 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/ae733f5f7a8e4c42bb82adb85845907b is 50, key is test_row_0/B:col10/1731631836343/Put/seqid=0 2024-11-15T00:50:36,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742327_1503 (size=12301) 2024-11-15T00:50:36,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/ae733f5f7a8e4c42bb82adb85845907b 2024-11-15T00:50:36,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/e440697e875546c8a35a5867a2dfcbf9 is 50, key is test_row_0/C:col10/1731631836343/Put/seqid=0 2024-11-15T00:50:36,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742328_1504 (size=12301) 2024-11-15T00:50:36,946 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T00:50:36,946 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:36,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:36,947 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:36,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:37,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631897017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,022 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631897018, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,023 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 153 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631897030, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,035 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T00:50:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:37,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:37,106 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:37,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:37,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:37,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T00:50:37,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T00:50:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:37,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:37,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:37,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:37,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=430 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/e440697e875546c8a35a5867a2dfcbf9 2024-11-15T00:50:37,340 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/a0edb668681e4fceb94376781f6bc5cf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a0edb668681e4fceb94376781f6bc5cf 2024-11-15T00:50:37,358 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a0edb668681e4fceb94376781f6bc5cf, entries=250, sequenceid=430, filesize=16.8 K 2024-11-15T00:50:37,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/ae733f5f7a8e4c42bb82adb85845907b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ae733f5f7a8e4c42bb82adb85845907b 2024-11-15T00:50:37,395 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ae733f5f7a8e4c42bb82adb85845907b, entries=150, sequenceid=430, filesize=12.0 K 2024-11-15T00:50:37,398 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/e440697e875546c8a35a5867a2dfcbf9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/e440697e875546c8a35a5867a2dfcbf9 2024-11-15T00:50:37,405 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/e440697e875546c8a35a5867a2dfcbf9, entries=150, sequenceid=430, filesize=12.0 K 2024-11-15T00:50:37,406 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 0015d049952a9227751d617c5d05c387 in 1062ms, sequenceid=430, compaction requested=true 2024-11-15T00:50:37,406 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:37,407 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:37,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:37,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:37,407 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:37,407 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:37,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:37,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:37,408 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:37,408 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 45177 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:37,408 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:37,408 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:37,408 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/1ae8f47bea4441a0aa878d9b6e5665c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/dd0109047f634bd4ba0b7988db24785b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a0edb668681e4fceb94376781f6bc5cf] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=44.1 K 2024-11-15T00:50:37,409 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:37,409 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:37,409 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:37,409 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/92b113c2cb6f4b439a89b19c738b6e45, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8e3e0398fbdb48fb9a872e7b77ff6d6a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ae733f5f7a8e4c42bb82adb85845907b] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=37.0 K 2024-11-15T00:50:37,409 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1ae8f47bea4441a0aa878d9b6e5665c0, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731631835306 2024-11-15T00:50:37,409 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 92b113c2cb6f4b439a89b19c738b6e45, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731631835306 2024-11-15T00:50:37,409 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting dd0109047f634bd4ba0b7988db24785b, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731631835449 2024-11-15T00:50:37,409 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e3e0398fbdb48fb9a872e7b77ff6d6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731631835464 2024-11-15T00:50:37,410 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0edb668681e4fceb94376781f6bc5cf, keycount=250, bloomtype=ROW, size=16.8 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731631836178 2024-11-15T00:50:37,410 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ae733f5f7a8e4c42bb82adb85845907b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731631836210 2024-11-15T00:50:37,413 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-15T00:50:37,416 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:37,416 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-15T00:50:37,427 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/7847c0a3f0784f82875964a1099d4c04 is 50, key is test_row_0/A:col10/1731631836377/Put/seqid=0 2024-11-15T00:50:37,444 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#428 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:37,445 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/99da3a1558294e8883685d6ce244e9c8 is 50, key is test_row_0/B:col10/1731631836343/Put/seqid=0 2024-11-15T00:50:37,453 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:37,454 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/b7941a2186d741b48830278de4108007 is 50, key is test_row_0/A:col10/1731631836343/Put/seqid=0 2024-11-15T00:50:37,491 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742329_1505 (size=12301) 2024-11-15T00:50:37,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742330_1506 (size=13357) 2024-11-15T00:50:37,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742331_1507 (size=13357) 2024-11-15T00:50:37,535 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:37,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:37,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631897637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,648 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631897637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,651 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631897647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,764 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631897757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631897757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,765 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631897757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,765 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,893 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/7847c0a3f0784f82875964a1099d4c04 2024-11-15T00:50:37,907 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/b7941a2186d741b48830278de4108007 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b7941a2186d741b48830278de4108007 2024-11-15T00:50:37,916 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into b7941a2186d741b48830278de4108007(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:37,916 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:37,916 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=13, startTime=1731631837407; duration=0sec 2024-11-15T00:50:37,916 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:37,916 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:37,916 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:37,925 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37857 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:37,925 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:37,925 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:37,925 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/14715df5d07244388f8665e447a7758a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d7c8a71c92b546a6b03447da1bfbd35c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/e440697e875546c8a35a5867a2dfcbf9] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=37.0 K 2024-11-15T00:50:37,929 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14715df5d07244388f8665e447a7758a, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=389, earliestPutTs=1731631835306 2024-11-15T00:50:37,929 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d7c8a71c92b546a6b03447da1bfbd35c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=404, earliestPutTs=1731631835464 2024-11-15T00:50:37,930 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e440697e875546c8a35a5867a2dfcbf9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731631836210 2024-11-15T00:50:37,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/c482ca37f03a46219c235a1ac07a6ab2 is 50, key is test_row_0/B:col10/1731631836377/Put/seqid=0 2024-11-15T00:50:37,940 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/99da3a1558294e8883685d6ce244e9c8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/99da3a1558294e8883685d6ce244e9c8 2024-11-15T00:50:37,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742332_1508 (size=12301) 2024-11-15T00:50:37,968 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/c482ca37f03a46219c235a1ac07a6ab2 2024-11-15T00:50:37,969 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into 99da3a1558294e8883685d6ce244e9c8(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:37,969 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:37,969 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=13, startTime=1731631837407; duration=0sec 2024-11-15T00:50:37,970 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:37,970 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:37,970 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#431 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:37,970 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/a6ba794a09a345219bb1e1299fca9f55 is 50, key is test_row_0/C:col10/1731631836343/Put/seqid=0 2024-11-15T00:50:37,980 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631897967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,980 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631897977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,982 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:37,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631897977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:37,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/9b639bb4ab1e410e8c20a2f0b0ded9a0 is 50, key is test_row_0/C:col10/1731631836377/Put/seqid=0 2024-11-15T00:50:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,983 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:37,984 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:37,985 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742333_1509 (size=13357) 2024-11-15T00:50:37,989 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/a6ba794a09a345219bb1e1299fca9f55 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a6ba794a09a345219bb1e1299fca9f55 2024-11-15T00:50:37,990 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742334_1510 (size=12301) 2024-11-15T00:50:37,993 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=440 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/9b639bb4ab1e410e8c20a2f0b0ded9a0 2024-11-15T00:50:37,999 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into a6ba794a09a345219bb1e1299fca9f55(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:38,000 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:38,000 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=13, startTime=1731631837408; duration=0sec 2024-11-15T00:50:38,000 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:38,000 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:38,001 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/7847c0a3f0784f82875964a1099d4c04 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/7847c0a3f0784f82875964a1099d4c04 2024-11-15T00:50:38,008 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/7847c0a3f0784f82875964a1099d4c04, entries=150, sequenceid=440, filesize=12.0 K 2024-11-15T00:50:38,008 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/c482ca37f03a46219c235a1ac07a6ab2 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c482ca37f03a46219c235a1ac07a6ab2 2024-11-15T00:50:38,012 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c482ca37f03a46219c235a1ac07a6ab2, entries=150, sequenceid=440, filesize=12.0 K 2024-11-15T00:50:38,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/9b639bb4ab1e410e8c20a2f0b0ded9a0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9b639bb4ab1e410e8c20a2f0b0ded9a0 2024-11-15T00:50:38,017 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9b639bb4ab1e410e8c20a2f0b0ded9a0, entries=150, sequenceid=440, filesize=12.0 K 2024-11-15T00:50:38,018 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 0015d049952a9227751d617c5d05c387 in 602ms, sequenceid=440, compaction requested=false 2024-11-15T00:50:38,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:38,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,019 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-15T00:50:38,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-15T00:50:38,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-15T00:50:38,032 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0370 sec 2024-11-15T00:50:38,033 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.0450 sec 2024-11-15T00:50:38,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-15T00:50:38,147 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:38,148 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees 2024-11-15T00:50:38,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T00:50:38,150 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:38,151 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=137, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:38,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:38,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T00:50:38,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:38,294 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:38,294 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-15T00:50:38,299 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/9a12aecfafb64962b244bff068cc4a87 is 50, key is test_row_0/A:col10/1731631837635/Put/seqid=0 2024-11-15T00:50:38,304 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T00:50:38,308 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742335_1511 (size=14741) 2024-11-15T00:50:38,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:38,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,314 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,324 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631898315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 165 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631898315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,325 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631898321, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,325 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,327 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 166 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631898437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,444 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631898437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,445 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,445 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631898440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,447 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56256 deadline: 1731631898446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,455 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18342 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,456 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T00:50:38,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T00:50:38,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:38,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,470 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,499 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56222 deadline: 1731631898497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,499 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 0015d049952a9227751d617c5d05c387 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18398 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:50:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T00:50:38,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:38,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 167 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631898656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631898657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,664 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,664 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631898661, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,665 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,709 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/9a12aecfafb64962b244bff068cc4a87 2024-11-15T00:50:38,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/aeba4a2927464d80b1ac493e4acaf60b is 50, key is test_row_0/B:col10/1731631837635/Put/seqid=0 2024-11-15T00:50:38,758 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742336_1512 (size=12301) 2024-11-15T00:50:38,779 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T00:50:38,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:38,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,782 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,782 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T00:50:38,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,809 INFO [Thread-1870 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1870. 2024-11-15T00:50:38,809 DEBUG [Thread-1870 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:38,809 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:38,809 DEBUG [Thread-1870 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:38,809 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:38,817 INFO [Thread-1876 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1876. 2024-11-15T00:50:38,817 DEBUG [Thread-1876 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:38,817 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:38,817 DEBUG [Thread-1876 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:38,817 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:38,821 INFO [Thread-1872 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1872. 2024-11-15T00:50:38,821 DEBUG [Thread-1872 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:38,822 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:38,823 INFO [Thread-1874 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1874. 2024-11-15T00:50:38,823 DEBUG [Thread-1874 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:38,823 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:38,823 DEBUG [Thread-1874 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:38,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:38,824 DEBUG [Thread-1872 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:38,824 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:38,827 INFO [Thread-1878 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1878. 2024-11-15T00:50:38,827 DEBUG [Thread-1878 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:38,827 DEBUG [Thread-1878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:38,827 DEBUG [Thread-1878 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:38,827 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:38,951 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T00:50:38,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:38,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:38,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631898977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 168 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631898977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631898977, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:38,978 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:39,104 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T00:50:39,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:39,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,104 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,104 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,105 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,159 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/aeba4a2927464d80b1ac493e4acaf60b 2024-11-15T00:50:39,167 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/d442f99a654441838355241479549f83 is 50, key is test_row_0/C:col10/1731631837635/Put/seqid=0 2024-11-15T00:50:39,172 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742337_1513 (size=12301) 2024-11-15T00:50:39,257 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T00:50:39,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:39,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,257 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T00:50:39,410 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T00:50:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,410 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,410 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:39,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:39,486 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56276 deadline: 1731631899486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 169 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56288 deadline: 1731631899486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:39,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:56196 deadline: 1731631899486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 , the old value is region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:39,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=2 because the exception is null or not the one we care about 2024-11-15T00:50:39,563 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T00:50:39,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:39,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,563 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] handler.RSProcedureHandler(58): pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=138 java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=138 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:39,572 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/d442f99a654441838355241479549f83 2024-11-15T00:50:39,576 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/9a12aecfafb64962b244bff068cc4a87 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9a12aecfafb64962b244bff068cc4a87 2024-11-15T00:50:39,579 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9a12aecfafb64962b244bff068cc4a87, entries=200, sequenceid=470, filesize=14.4 K 2024-11-15T00:50:39,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/aeba4a2927464d80b1ac493e4acaf60b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/aeba4a2927464d80b1ac493e4acaf60b 2024-11-15T00:50:39,582 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/aeba4a2927464d80b1ac493e4acaf60b, entries=150, sequenceid=470, filesize=12.0 K 2024-11-15T00:50:39,583 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/d442f99a654441838355241479549f83 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d442f99a654441838355241479549f83 2024-11-15T00:50:39,585 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d442f99a654441838355241479549f83, entries=150, sequenceid=470, filesize=12.0 K 2024-11-15T00:50:39,586 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 0015d049952a9227751d617c5d05c387 in 1292ms, sequenceid=470, compaction requested=true 2024-11-15T00:50:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:39,586 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:39,586 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 0015d049952a9227751d617c5d05c387:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:39,586 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:39,587 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40399 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:39,587 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:39,587 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/B is initiating minor compaction (all files) 2024-11-15T00:50:39,587 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/A is initiating minor compaction (all files) 2024-11-15T00:50:39,587 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/B in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,587 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/A in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,587 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b7941a2186d741b48830278de4108007, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/7847c0a3f0784f82875964a1099d4c04, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9a12aecfafb64962b244bff068cc4a87] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=39.5 K 2024-11-15T00:50:39,587 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/99da3a1558294e8883685d6ce244e9c8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c482ca37f03a46219c235a1ac07a6ab2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/aeba4a2927464d80b1ac493e4acaf60b] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=37.1 K 2024-11-15T00:50:39,587 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 99da3a1558294e8883685d6ce244e9c8, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731631836210 2024-11-15T00:50:39,587 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b7941a2186d741b48830278de4108007, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731631836210 2024-11-15T00:50:39,587 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7847c0a3f0784f82875964a1099d4c04, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731631836364 2024-11-15T00:50:39,587 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting c482ca37f03a46219c235a1ac07a6ab2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731631836364 2024-11-15T00:50:39,588 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting aeba4a2927464d80b1ac493e4acaf60b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731631837633 2024-11-15T00:50:39,588 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9a12aecfafb64962b244bff068cc4a87, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731631837633 2024-11-15T00:50:39,593 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#B#compaction#436 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:39,593 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/c20f8dd5a2884bfe8d3fded2652a7937 is 50, key is test_row_0/B:col10/1731631837635/Put/seqid=0 2024-11-15T00:50:39,593 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#A#compaction#437 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:39,594 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/0547f2c62ed0484b85cdf6c4d99dbc68 is 50, key is test_row_0/A:col10/1731631837635/Put/seqid=0 2024-11-15T00:50:39,597 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742338_1514 (size=13459) 2024-11-15T00:50:39,598 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742339_1515 (size=13459) 2024-11-15T00:50:39,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=138 2024-11-15T00:50:39,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:39,716 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-15T00:50:39,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/0277696bc0b947f39dd05ceebbe302b8 is 50, key is test_row_0/A:col10/1731631838303/Put/seqid=0 2024-11-15T00:50:39,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742340_1516 (size=12301) 2024-11-15T00:50:40,002 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/0547f2c62ed0484b85cdf6c4d99dbc68 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/0547f2c62ed0484b85cdf6c4d99dbc68 2024-11-15T00:50:40,002 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/c20f8dd5a2884bfe8d3fded2652a7937 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c20f8dd5a2884bfe8d3fded2652a7937 2024-11-15T00:50:40,007 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/B of 0015d049952a9227751d617c5d05c387 into c20f8dd5a2884bfe8d3fded2652a7937(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:40,007 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:40,007 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/B, priority=13, startTime=1731631839586; duration=0sec 2024-11-15T00:50:40,008 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:40,008 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:B 2024-11-15T00:50:40,008 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:40,008 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/A of 0015d049952a9227751d617c5d05c387 into 0547f2c62ed0484b85cdf6c4d99dbc68(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:40,008 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:40,008 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/A, priority=13, startTime=1731631839586; duration=0sec 2024-11-15T00:50:40,008 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:40,008 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:A 2024-11-15T00:50:40,009 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37959 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:40,009 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 0015d049952a9227751d617c5d05c387/C is initiating minor compaction (all files) 2024-11-15T00:50:40,009 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 0015d049952a9227751d617c5d05c387/C in TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:40,009 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a6ba794a09a345219bb1e1299fca9f55, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9b639bb4ab1e410e8c20a2f0b0ded9a0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d442f99a654441838355241479549f83] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp, totalSize=37.1 K 2024-11-15T00:50:40,009 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a6ba794a09a345219bb1e1299fca9f55, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=430, earliestPutTs=1731631836210 2024-11-15T00:50:40,010 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b639bb4ab1e410e8c20a2f0b0ded9a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=440, earliestPutTs=1731631836364 2024-11-15T00:50:40,010 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting d442f99a654441838355241479549f83, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1731631837633 2024-11-15T00:50:40,018 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 0015d049952a9227751d617c5d05c387#C#compaction#439 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:40,019 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/ef3530a1297a41de873d6032089653d8 is 50, key is test_row_0/C:col10/1731631837635/Put/seqid=0 2024-11-15T00:50:40,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742341_1517 (size=13459) 2024-11-15T00:50:40,122 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/0277696bc0b947f39dd05ceebbe302b8 2024-11-15T00:50:40,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/e698c2afae74416c8b5358aaf73b7207 is 50, key is test_row_0/B:col10/1731631838303/Put/seqid=0 2024-11-15T00:50:40,133 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742342_1518 (size=12301) 2024-11-15T00:50:40,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T00:50:40,432 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/ef3530a1297a41de873d6032089653d8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/ef3530a1297a41de873d6032089653d8 2024-11-15T00:50:40,437 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 0015d049952a9227751d617c5d05c387/C of 0015d049952a9227751d617c5d05c387 into ef3530a1297a41de873d6032089653d8(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:40,437 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:40,437 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., storeName=0015d049952a9227751d617c5d05c387/C, priority=13, startTime=1731631839586; duration=0sec 2024-11-15T00:50:40,437 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:40,437 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 0015d049952a9227751d617c5d05c387:C 2024-11-15T00:50:40,501 INFO [Thread-1859 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1859. 2024-11-15T00:50:40,501 DEBUG [Thread-1859 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:40,501 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:40,501 DEBUG [Thread-1859 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:40,501 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:40,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:40,540 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. as already flushing 2024-11-15T00:50:40,540 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/e698c2afae74416c8b5358aaf73b7207 2024-11-15T00:50:40,541 INFO [Thread-1861 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1861. 2024-11-15T00:50:40,541 DEBUG [Thread-1861 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:40,541 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:40,541 DEBUG [Thread-1861 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:40,544 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:40,546 INFO [Thread-1867 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1867. 2024-11-15T00:50:40,546 DEBUG [Thread-1867 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:40,546 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:40,546 DEBUG [Thread-1867 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:40,560 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:40,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/44410571e1d045a8943224557815c621 is 50, key is test_row_0/C:col10/1731631838303/Put/seqid=0 2024-11-15T00:50:40,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742343_1519 (size=12301) 2024-11-15T00:50:40,978 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=479 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/44410571e1d045a8943224557815c621 2024-11-15T00:50:40,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/0277696bc0b947f39dd05ceebbe302b8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/0277696bc0b947f39dd05ceebbe302b8 2024-11-15T00:50:40,985 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/0277696bc0b947f39dd05ceebbe302b8, entries=150, sequenceid=479, filesize=12.0 K 2024-11-15T00:50:40,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/e698c2afae74416c8b5358aaf73b7207 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/e698c2afae74416c8b5358aaf73b7207 2024-11-15T00:50:40,989 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/e698c2afae74416c8b5358aaf73b7207, entries=150, sequenceid=479, filesize=12.0 K 2024-11-15T00:50:40,990 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/44410571e1d045a8943224557815c621 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/44410571e1d045a8943224557815c621 2024-11-15T00:50:40,993 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/44410571e1d045a8943224557815c621, entries=150, sequenceid=479, filesize=12.0 K 2024-11-15T00:50:40,995 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=20.13 KB/20610 for 0015d049952a9227751d617c5d05c387 in 1278ms, sequenceid=479, compaction requested=false 2024-11-15T00:50:40,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.HRegion(2603): Flush status journal for 0015d049952a9227751d617c5d05c387: 2024-11-15T00:50:40,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:40,995 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=138}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=138 2024-11-15T00:50:40,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=138 2024-11-15T00:50:40,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-15T00:50:40,998 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.8460 sec 2024-11-15T00:50:40,999 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=137, table=TestAcidGuarantees in 2.8510 sec 2024-11-15T00:50:42,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-15T00:50:42,317 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:46,788 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:50:48,478 INFO [Thread-1865 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1865. 2024-11-15T00:50:48,478 DEBUG [Thread-1865 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:48,478 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:48,478 DEBUG [Thread-1865 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:48,478 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:48,568 INFO [Thread-1863 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1863. 2024-11-15T00:50:48,568 DEBUG [Thread-1863 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:50:48,568 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:48,568 DEBUG [Thread-1863 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:48,568 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 132 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 115 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 9 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 113 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 979 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2937 rows 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 989 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2967 rows 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 988 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2964 rows 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 990 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2970 rows 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 985 2024-11-15T00:50:48,569 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 2954 rows 2024-11-15T00:50:48,569 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:50:48,569 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:50:48,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:48,569 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:48,569 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:50:48,570 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:50:48,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=139, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T00:50:48,577 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631848576"}]},"ts":"1731631848576"} 2024-11-15T00:50:48,578 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:50:48,578 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:50:48,580 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:50:48,587 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0015d049952a9227751d617c5d05c387, UNASSIGN}] 2024-11-15T00:50:48,588 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0015d049952a9227751d617c5d05c387, UNASSIGN 2024-11-15T00:50:48,588 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=0015d049952a9227751d617c5d05c387, regionState=CLOSING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:48,593 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=141, ppid=140, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=0015d049952a9227751d617c5d05c387, UNASSIGN because future has completed 2024-11-15T00:50:48,593 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:50:48,593 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:50:48,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T00:50:48,751 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(122): Close 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:48,751 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:50:48,751 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1722): Closing 0015d049952a9227751d617c5d05c387, disabling compactions & flushes 2024-11-15T00:50:48,751 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:48,751 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:48,751 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. after waiting 0 ms 2024-11-15T00:50:48,751 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:48,752 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(2902): Flushing 0015d049952a9227751d617c5d05c387 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T00:50:48,755 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/ed747f7f812c44f1a60afd6a21dafe7b is 50, key is test_row_0/A:col10/1731631840507/Put/seqid=0 2024-11-15T00:50:48,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742344_1520 (size=12301) 2024-11-15T00:50:48,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T00:50:49,158 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/ed747f7f812c44f1a60afd6a21dafe7b 2024-11-15T00:50:49,167 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/fb290ffd02394a07b7fde578d950a983 is 50, key is test_row_0/B:col10/1731631840507/Put/seqid=0 2024-11-15T00:50:49,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742345_1521 (size=12301) 2024-11-15T00:50:49,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T00:50:49,586 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/fb290ffd02394a07b7fde578d950a983 2024-11-15T00:50:49,594 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/7f6355d4edeb432f830f668508d4752a is 50, key is test_row_0/C:col10/1731631840507/Put/seqid=0 2024-11-15T00:50:49,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742346_1522 (size=12301) 2024-11-15T00:50:49,614 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=490 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/7f6355d4edeb432f830f668508d4752a 2024-11-15T00:50:49,623 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/A/ed747f7f812c44f1a60afd6a21dafe7b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/ed747f7f812c44f1a60afd6a21dafe7b 2024-11-15T00:50:49,628 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/ed747f7f812c44f1a60afd6a21dafe7b, entries=150, sequenceid=490, filesize=12.0 K 2024-11-15T00:50:49,629 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/B/fb290ffd02394a07b7fde578d950a983 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/fb290ffd02394a07b7fde578d950a983 2024-11-15T00:50:49,634 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/fb290ffd02394a07b7fde578d950a983, entries=150, sequenceid=490, filesize=12.0 K 2024-11-15T00:50:49,635 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/.tmp/C/7f6355d4edeb432f830f668508d4752a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7f6355d4edeb432f830f668508d4752a 2024-11-15T00:50:49,639 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7f6355d4edeb432f830f668508d4752a, entries=150, sequenceid=490, filesize=12.0 K 2024-11-15T00:50:49,643 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0015d049952a9227751d617c5d05c387 in 891ms, sequenceid=490, compaction requested=true 2024-11-15T00:50:49,643 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/f95463774b3d4bdd8cc41357bbb51333, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a80f13c573244ed0ad91da6b7ee1e984, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/04485c2cb151428b8fd477145c09f2dd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b25937f9946e4593a9ebe83f2739e6a9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/10b00a3b2d88416391d3b0946b045ef1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/de7bc3b0da604da6a699e7cfa1a4d140, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9405c8433512455cadc5dfe00056c449, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b8745c01fd974e139ddd30e16163eab1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/746b4672f0e8411bba395919d377d528, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/03dd4ef125a04bcfac6c5ab4e1d3da1f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/5cad0eafc3604417983ff825280569e2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c8e3ba7e6f9a46f9867f59474c0038bd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/4542c0ce6b3e4205978a373c019fda66, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a8ae1bc687fd429a8a80c00036d3b234, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/cf1ca71f97cf4e138185ede93ad32d70, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/67e77bf512b64e91b2194782e37c923c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/817e1ea9198549b3836f84cc7416ef78, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c7d0c48efec540ba8b4dcb7750bacc4a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/3aec4f3239db4eeeb0e156ec9e0ade43, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9170d529bb7d4d9b9455ef2a45d42b53, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2cd31791f5494ef5a2e45b341068db39, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/110bdcfb112e4f2ea17274f008066e94, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/159a5784dcb240bd8628ee2c499e0b02, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c69e685c5710479f9965530a224525ff, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/37fa7a0f0c6545d9a27835a7ef6ef460, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2b2a3d8a5bb549fab222a4ec35f1b6f8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2ad7f4d32a6546d3b6ecba8950463127, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/190ccb02eb584aa484ca84439554ddc2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/1ae8f47bea4441a0aa878d9b6e5665c0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/dd0109047f634bd4ba0b7988db24785b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a0edb668681e4fceb94376781f6bc5cf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b7941a2186d741b48830278de4108007, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/7847c0a3f0784f82875964a1099d4c04, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9a12aecfafb64962b244bff068cc4a87] to archive 2024-11-15T00:50:49,645 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:50:49,650 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/f95463774b3d4bdd8cc41357bbb51333 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/f95463774b3d4bdd8cc41357bbb51333 2024-11-15T00:50:49,651 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a80f13c573244ed0ad91da6b7ee1e984 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a80f13c573244ed0ad91da6b7ee1e984 2024-11-15T00:50:49,655 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/04485c2cb151428b8fd477145c09f2dd to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/04485c2cb151428b8fd477145c09f2dd 2024-11-15T00:50:49,661 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b25937f9946e4593a9ebe83f2739e6a9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b25937f9946e4593a9ebe83f2739e6a9 2024-11-15T00:50:49,662 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/10b00a3b2d88416391d3b0946b045ef1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/10b00a3b2d88416391d3b0946b045ef1 2024-11-15T00:50:49,669 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/de7bc3b0da604da6a699e7cfa1a4d140 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/de7bc3b0da604da6a699e7cfa1a4d140 2024-11-15T00:50:49,673 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9405c8433512455cadc5dfe00056c449 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9405c8433512455cadc5dfe00056c449 2024-11-15T00:50:49,674 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b8745c01fd974e139ddd30e16163eab1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b8745c01fd974e139ddd30e16163eab1 2024-11-15T00:50:49,677 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/746b4672f0e8411bba395919d377d528 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/746b4672f0e8411bba395919d377d528 2024-11-15T00:50:49,692 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/03dd4ef125a04bcfac6c5ab4e1d3da1f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/03dd4ef125a04bcfac6c5ab4e1d3da1f 2024-11-15T00:50:49,697 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/5cad0eafc3604417983ff825280569e2 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/5cad0eafc3604417983ff825280569e2 2024-11-15T00:50:49,700 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c8e3ba7e6f9a46f9867f59474c0038bd to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c8e3ba7e6f9a46f9867f59474c0038bd 2024-11-15T00:50:49,702 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/4542c0ce6b3e4205978a373c019fda66 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/4542c0ce6b3e4205978a373c019fda66 2024-11-15T00:50:49,704 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a8ae1bc687fd429a8a80c00036d3b234 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a8ae1bc687fd429a8a80c00036d3b234 2024-11-15T00:50:49,705 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/cf1ca71f97cf4e138185ede93ad32d70 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/cf1ca71f97cf4e138185ede93ad32d70 2024-11-15T00:50:49,706 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/67e77bf512b64e91b2194782e37c923c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/67e77bf512b64e91b2194782e37c923c 2024-11-15T00:50:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T00:50:49,707 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/817e1ea9198549b3836f84cc7416ef78 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/817e1ea9198549b3836f84cc7416ef78 2024-11-15T00:50:49,709 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c7d0c48efec540ba8b4dcb7750bacc4a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c7d0c48efec540ba8b4dcb7750bacc4a 2024-11-15T00:50:49,710 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/3aec4f3239db4eeeb0e156ec9e0ade43 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/3aec4f3239db4eeeb0e156ec9e0ade43 2024-11-15T00:50:49,711 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9170d529bb7d4d9b9455ef2a45d42b53 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9170d529bb7d4d9b9455ef2a45d42b53 2024-11-15T00:50:49,712 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2cd31791f5494ef5a2e45b341068db39 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2cd31791f5494ef5a2e45b341068db39 2024-11-15T00:50:49,714 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/110bdcfb112e4f2ea17274f008066e94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/110bdcfb112e4f2ea17274f008066e94 2024-11-15T00:50:49,719 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/159a5784dcb240bd8628ee2c499e0b02 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/159a5784dcb240bd8628ee2c499e0b02 2024-11-15T00:50:49,721 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c69e685c5710479f9965530a224525ff to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/c69e685c5710479f9965530a224525ff 2024-11-15T00:50:49,723 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/37fa7a0f0c6545d9a27835a7ef6ef460 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/37fa7a0f0c6545d9a27835a7ef6ef460 2024-11-15T00:50:49,725 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2b2a3d8a5bb549fab222a4ec35f1b6f8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2b2a3d8a5bb549fab222a4ec35f1b6f8 2024-11-15T00:50:49,727 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2ad7f4d32a6546d3b6ecba8950463127 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/2ad7f4d32a6546d3b6ecba8950463127 2024-11-15T00:50:49,728 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/190ccb02eb584aa484ca84439554ddc2 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/190ccb02eb584aa484ca84439554ddc2 2024-11-15T00:50:49,729 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/1ae8f47bea4441a0aa878d9b6e5665c0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/1ae8f47bea4441a0aa878d9b6e5665c0 2024-11-15T00:50:49,730 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/dd0109047f634bd4ba0b7988db24785b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/dd0109047f634bd4ba0b7988db24785b 2024-11-15T00:50:49,731 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a0edb668681e4fceb94376781f6bc5cf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/a0edb668681e4fceb94376781f6bc5cf 2024-11-15T00:50:49,735 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b7941a2186d741b48830278de4108007 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/b7941a2186d741b48830278de4108007 2024-11-15T00:50:49,737 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/7847c0a3f0784f82875964a1099d4c04 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/7847c0a3f0784f82875964a1099d4c04 2024-11-15T00:50:49,738 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9a12aecfafb64962b244bff068cc4a87 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/9a12aecfafb64962b244bff068cc4a87 2024-11-15T00:50:49,740 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ac23a2b888fa44209b5bb25f7b7924ea, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28bc70ca1e1d4e33ad6c0962ceaa1aa9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/1538bc22619f41e2b24c9685d0524be6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/d41afd629f624a068907e943af9fab91, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bd4ce87da29042a981892ca764f01528, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/663ada3f688446eb92422b0691730886, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/6fcba9767ae14d3b95553c6ac789c3e9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/b7b6362b8ae5483fb5cdc82186305718, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/55a6931c3a3d475186976e27f7aa150f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/15cfec22d638447f99a098ba7b8ac866, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/50f0dd3f18344056a78f9445e42c40df, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8c6a911a7aca46d3b69ac1c019ba1243, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/03790e061c8e4953847dcafa6fa068c4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/012234564ea24e6aafbaaadf36f61e3c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f5a8a71d4fd541db8bbf61e65d812925, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/abc6f9065a45404586cf6c67554ae3f9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28901ceeb2d24ce69b9fce84f4e0de71, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f4a96be1a2894c99a6925a985eb34b47, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c5f52b46bbcf44caab11cae788329d9e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bdbf164fc8bf43e6baa95020220b01ee, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f91463ef257849e796fdb100014598e9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/2815edce48dc4365bec31e17f5ca42d0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/91525edb582441d28bffdad55458ab5b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/19eec3197d5d4be1921f8a68796c1cab, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8acb22a8de064603898724c0ec6ef1ec, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c68275fef6ef4676947e73928f5dc5b3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28a0da45e427454ca941f3b3c3f5d45a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/92b113c2cb6f4b439a89b19c738b6e45, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/3eb9b9c18db0427791f675c512783814, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8e3e0398fbdb48fb9a872e7b77ff6d6a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/99da3a1558294e8883685d6ce244e9c8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ae733f5f7a8e4c42bb82adb85845907b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c482ca37f03a46219c235a1ac07a6ab2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/aeba4a2927464d80b1ac493e4acaf60b] to archive 2024-11-15T00:50:49,742 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:50:49,748 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ac23a2b888fa44209b5bb25f7b7924ea to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ac23a2b888fa44209b5bb25f7b7924ea 2024-11-15T00:50:49,758 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28bc70ca1e1d4e33ad6c0962ceaa1aa9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28bc70ca1e1d4e33ad6c0962ceaa1aa9 2024-11-15T00:50:49,760 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/1538bc22619f41e2b24c9685d0524be6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/1538bc22619f41e2b24c9685d0524be6 2024-11-15T00:50:49,768 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/d41afd629f624a068907e943af9fab91 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/d41afd629f624a068907e943af9fab91 2024-11-15T00:50:49,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bd4ce87da29042a981892ca764f01528 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bd4ce87da29042a981892ca764f01528 2024-11-15T00:50:49,775 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/663ada3f688446eb92422b0691730886 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/663ada3f688446eb92422b0691730886 2024-11-15T00:50:49,776 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/6fcba9767ae14d3b95553c6ac789c3e9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/6fcba9767ae14d3b95553c6ac789c3e9 2024-11-15T00:50:49,777 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/b7b6362b8ae5483fb5cdc82186305718 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/b7b6362b8ae5483fb5cdc82186305718 2024-11-15T00:50:49,779 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/55a6931c3a3d475186976e27f7aa150f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/55a6931c3a3d475186976e27f7aa150f 2024-11-15T00:50:49,780 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/15cfec22d638447f99a098ba7b8ac866 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/15cfec22d638447f99a098ba7b8ac866 2024-11-15T00:50:49,782 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/50f0dd3f18344056a78f9445e42c40df to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/50f0dd3f18344056a78f9445e42c40df 2024-11-15T00:50:49,784 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8c6a911a7aca46d3b69ac1c019ba1243 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8c6a911a7aca46d3b69ac1c019ba1243 2024-11-15T00:50:49,785 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/03790e061c8e4953847dcafa6fa068c4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/03790e061c8e4953847dcafa6fa068c4 2024-11-15T00:50:49,786 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/012234564ea24e6aafbaaadf36f61e3c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/012234564ea24e6aafbaaadf36f61e3c 2024-11-15T00:50:49,787 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f5a8a71d4fd541db8bbf61e65d812925 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f5a8a71d4fd541db8bbf61e65d812925 2024-11-15T00:50:49,788 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/abc6f9065a45404586cf6c67554ae3f9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/abc6f9065a45404586cf6c67554ae3f9 2024-11-15T00:50:49,791 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28901ceeb2d24ce69b9fce84f4e0de71 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28901ceeb2d24ce69b9fce84f4e0de71 2024-11-15T00:50:49,792 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f4a96be1a2894c99a6925a985eb34b47 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f4a96be1a2894c99a6925a985eb34b47 2024-11-15T00:50:49,793 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c5f52b46bbcf44caab11cae788329d9e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c5f52b46bbcf44caab11cae788329d9e 2024-11-15T00:50:49,796 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bdbf164fc8bf43e6baa95020220b01ee to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/bdbf164fc8bf43e6baa95020220b01ee 2024-11-15T00:50:49,797 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f91463ef257849e796fdb100014598e9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/f91463ef257849e796fdb100014598e9 2024-11-15T00:50:49,798 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/2815edce48dc4365bec31e17f5ca42d0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/2815edce48dc4365bec31e17f5ca42d0 2024-11-15T00:50:49,799 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/91525edb582441d28bffdad55458ab5b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/91525edb582441d28bffdad55458ab5b 2024-11-15T00:50:49,800 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/19eec3197d5d4be1921f8a68796c1cab to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/19eec3197d5d4be1921f8a68796c1cab 2024-11-15T00:50:49,801 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8acb22a8de064603898724c0ec6ef1ec to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8acb22a8de064603898724c0ec6ef1ec 2024-11-15T00:50:49,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c68275fef6ef4676947e73928f5dc5b3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c68275fef6ef4676947e73928f5dc5b3 2024-11-15T00:50:49,803 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28a0da45e427454ca941f3b3c3f5d45a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/28a0da45e427454ca941f3b3c3f5d45a 2024-11-15T00:50:49,809 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/92b113c2cb6f4b439a89b19c738b6e45 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/92b113c2cb6f4b439a89b19c738b6e45 2024-11-15T00:50:49,811 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/3eb9b9c18db0427791f675c512783814 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/3eb9b9c18db0427791f675c512783814 2024-11-15T00:50:49,812 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8e3e0398fbdb48fb9a872e7b77ff6d6a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/8e3e0398fbdb48fb9a872e7b77ff6d6a 2024-11-15T00:50:49,813 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/99da3a1558294e8883685d6ce244e9c8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/99da3a1558294e8883685d6ce244e9c8 2024-11-15T00:50:49,817 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ae733f5f7a8e4c42bb82adb85845907b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/ae733f5f7a8e4c42bb82adb85845907b 2024-11-15T00:50:49,819 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c482ca37f03a46219c235a1ac07a6ab2 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c482ca37f03a46219c235a1ac07a6ab2 2024-11-15T00:50:49,821 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/aeba4a2927464d80b1ac493e4acaf60b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/aeba4a2927464d80b1ac493e4acaf60b 2024-11-15T00:50:49,822 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a029144b1fbd46159672b3983242cbd8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/f57ac424d0c64ef4ac32227b282fc88a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/dfd4bb3053a046bfa87871a51176e011, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0ae9c3e5d1e1495882132a0b4f3ba9eb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5339deaa331e4a36aa3f336223f9591b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/51b76cd0bacf4746ba6c4f3665fee2a9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a277d3bfcd9a4845bc337dac76a420c8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/b506a35151624aa28b9e16a197114619, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a5ca082436ce4bfd8785ee3c578d6c53, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9379bba82b5c47d6bb22989e5f548054, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5db27adb9d8b42b3a1670013c7edaad4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0578644c246e4ab0991b6c8af96a9c1e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6f4ab458376449f0a037254ef211c54e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5a5f9f0efee742c4a4d245412722ae35, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6263179dd09947bfbd77d3ceb9de65fe, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/ecaaec0477324aea9c08d1c5ca988ff2, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/071ea17f6d5143e1a8ff470fe0908e4d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/16d4b4fcf3cc4249b2d983ff618440b0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/43f5f1ae6cd94dc090e43b10b12e2ce6, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/4c15a5af5b704b69aa0d8f8751a833da, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/92592c4b04bd48f08089e7416fa5b23d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7bf1d554db584539a1d7a54eac935f54, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/417a0e3fd0c242a28cb9ee34b7f79c19, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/2ecab4a33181418399c00999da4d6c30, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/534637522b974283a782a260cb35ed6f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/29360e38c24f4de98ab4c212c0172e18, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7eaa6fbb28e24c8e85737a58dfb45f5f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/14715df5d07244388f8665e447a7758a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/df52c170996b475591ed6b7572dc972f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d7c8a71c92b546a6b03447da1bfbd35c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a6ba794a09a345219bb1e1299fca9f55, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/e440697e875546c8a35a5867a2dfcbf9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9b639bb4ab1e410e8c20a2f0b0ded9a0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d442f99a654441838355241479549f83] to archive 2024-11-15T00:50:49,823 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:50:49,824 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a029144b1fbd46159672b3983242cbd8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a029144b1fbd46159672b3983242cbd8 2024-11-15T00:50:49,825 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/f57ac424d0c64ef4ac32227b282fc88a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/f57ac424d0c64ef4ac32227b282fc88a 2024-11-15T00:50:49,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/dfd4bb3053a046bfa87871a51176e011 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/dfd4bb3053a046bfa87871a51176e011 2024-11-15T00:50:49,826 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0ae9c3e5d1e1495882132a0b4f3ba9eb to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0ae9c3e5d1e1495882132a0b4f3ba9eb 2024-11-15T00:50:49,828 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5339deaa331e4a36aa3f336223f9591b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5339deaa331e4a36aa3f336223f9591b 2024-11-15T00:50:49,829 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/51b76cd0bacf4746ba6c4f3665fee2a9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/51b76cd0bacf4746ba6c4f3665fee2a9 2024-11-15T00:50:49,835 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a277d3bfcd9a4845bc337dac76a420c8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a277d3bfcd9a4845bc337dac76a420c8 2024-11-15T00:50:49,836 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/b506a35151624aa28b9e16a197114619 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/b506a35151624aa28b9e16a197114619 2024-11-15T00:50:49,838 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a5ca082436ce4bfd8785ee3c578d6c53 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a5ca082436ce4bfd8785ee3c578d6c53 2024-11-15T00:50:49,839 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9379bba82b5c47d6bb22989e5f548054 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9379bba82b5c47d6bb22989e5f548054 2024-11-15T00:50:49,840 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5db27adb9d8b42b3a1670013c7edaad4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5db27adb9d8b42b3a1670013c7edaad4 2024-11-15T00:50:49,844 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0578644c246e4ab0991b6c8af96a9c1e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/0578644c246e4ab0991b6c8af96a9c1e 2024-11-15T00:50:49,846 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6f4ab458376449f0a037254ef211c54e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6f4ab458376449f0a037254ef211c54e 2024-11-15T00:50:49,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5a5f9f0efee742c4a4d245412722ae35 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/5a5f9f0efee742c4a4d245412722ae35 2024-11-15T00:50:49,847 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6263179dd09947bfbd77d3ceb9de65fe to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/6263179dd09947bfbd77d3ceb9de65fe 2024-11-15T00:50:49,848 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/ecaaec0477324aea9c08d1c5ca988ff2 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/ecaaec0477324aea9c08d1c5ca988ff2 2024-11-15T00:50:49,849 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/071ea17f6d5143e1a8ff470fe0908e4d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/071ea17f6d5143e1a8ff470fe0908e4d 2024-11-15T00:50:49,850 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/16d4b4fcf3cc4249b2d983ff618440b0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/16d4b4fcf3cc4249b2d983ff618440b0 2024-11-15T00:50:49,851 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/43f5f1ae6cd94dc090e43b10b12e2ce6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/43f5f1ae6cd94dc090e43b10b12e2ce6 2024-11-15T00:50:49,853 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/4c15a5af5b704b69aa0d8f8751a833da to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/4c15a5af5b704b69aa0d8f8751a833da 2024-11-15T00:50:49,856 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/92592c4b04bd48f08089e7416fa5b23d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/92592c4b04bd48f08089e7416fa5b23d 2024-11-15T00:50:49,857 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7bf1d554db584539a1d7a54eac935f54 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7bf1d554db584539a1d7a54eac935f54 2024-11-15T00:50:49,859 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/417a0e3fd0c242a28cb9ee34b7f79c19 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/417a0e3fd0c242a28cb9ee34b7f79c19 2024-11-15T00:50:49,860 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/2ecab4a33181418399c00999da4d6c30 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/2ecab4a33181418399c00999da4d6c30 2024-11-15T00:50:49,862 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/534637522b974283a782a260cb35ed6f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/534637522b974283a782a260cb35ed6f 2024-11-15T00:50:49,865 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/29360e38c24f4de98ab4c212c0172e18 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/29360e38c24f4de98ab4c212c0172e18 2024-11-15T00:50:49,866 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7eaa6fbb28e24c8e85737a58dfb45f5f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7eaa6fbb28e24c8e85737a58dfb45f5f 2024-11-15T00:50:49,867 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/14715df5d07244388f8665e447a7758a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/14715df5d07244388f8665e447a7758a 2024-11-15T00:50:49,869 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/df52c170996b475591ed6b7572dc972f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/df52c170996b475591ed6b7572dc972f 2024-11-15T00:50:49,870 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d7c8a71c92b546a6b03447da1bfbd35c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d7c8a71c92b546a6b03447da1bfbd35c 2024-11-15T00:50:49,871 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a6ba794a09a345219bb1e1299fca9f55 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/a6ba794a09a345219bb1e1299fca9f55 2024-11-15T00:50:49,873 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/e440697e875546c8a35a5867a2dfcbf9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/e440697e875546c8a35a5867a2dfcbf9 2024-11-15T00:50:49,874 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9b639bb4ab1e410e8c20a2f0b0ded9a0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/9b639bb4ab1e410e8c20a2f0b0ded9a0 2024-11-15T00:50:49,875 DEBUG [StoreCloser-TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d442f99a654441838355241479549f83 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/d442f99a654441838355241479549f83 2024-11-15T00:50:49,889 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/recovered.edits/493.seqid, newMaxSeqId=493, maxSeqId=1 2024-11-15T00:50:49,890 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. 2024-11-15T00:50:49,890 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] regionserver.HRegion(1676): Region close journal for 0015d049952a9227751d617c5d05c387: Waiting for close lock at 1731631848751Running coprocessor pre-close hooks at 1731631848751Disabling compacts and flushes for region at 1731631848751Disabling writes for close at 1731631848751Obtaining lock to block concurrent updates at 1731631848752 (+1 ms)Preparing flush snapshotting stores in 0015d049952a9227751d617c5d05c387 at 1731631848752Finished memstore snapshotting TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731631848752Flushing stores of TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387. at 1731631848752Flushing 0015d049952a9227751d617c5d05c387/A: creating writer at 1731631848752Flushing 0015d049952a9227751d617c5d05c387/A: appending metadata at 1731631848754 (+2 ms)Flushing 0015d049952a9227751d617c5d05c387/A: closing flushed file at 1731631848754Flushing 0015d049952a9227751d617c5d05c387/B: creating writer at 1731631849164 (+410 ms)Flushing 0015d049952a9227751d617c5d05c387/B: appending metadata at 1731631849166 (+2 ms)Flushing 0015d049952a9227751d617c5d05c387/B: closing flushed file at 1731631849166Flushing 0015d049952a9227751d617c5d05c387/C: creating writer at 1731631849591 (+425 ms)Flushing 0015d049952a9227751d617c5d05c387/C: appending metadata at 1731631849593 (+2 ms)Flushing 0015d049952a9227751d617c5d05c387/C: closing flushed file at 1731631849593Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@545f064f: reopening flushed file at 1731631849617 (+24 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@288bc967: reopening flushed file at 1731631849628 (+11 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1062785e: reopening flushed file at 1731631849635 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 0015d049952a9227751d617c5d05c387 in 891ms, sequenceid=490, compaction requested=true at 1731631849643 (+8 ms)Writing region close event to WAL at 1731631849876 (+233 ms)Running coprocessor post-close hooks at 1731631849890 (+14 ms)Closed at 1731631849890 2024-11-15T00:50:49,894 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=141 updating hbase:meta row=0015d049952a9227751d617c5d05c387, regionState=CLOSED 2024-11-15T00:50:49,896 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=142}] handler.UnassignRegionHandler(157): Closed 0015d049952a9227751d617c5d05c387 2024-11-15T00:50:49,896 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=142, ppid=141, state=RUNNABLE, hasLock=false; CloseRegionProcedure 0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:50:49,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=142, resume processing ppid=141 2024-11-15T00:50:49,899 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, ppid=141, state=SUCCESS, hasLock=false; CloseRegionProcedure 0015d049952a9227751d617c5d05c387, server=fb2f6cb7846a,42925,1731631698709 in 1.3050 sec 2024-11-15T00:50:49,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=141, resume processing ppid=140 2024-11-15T00:50:49,902 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, ppid=140, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=0015d049952a9227751d617c5d05c387, UNASSIGN in 1.3130 sec 2024-11-15T00:50:49,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-15T00:50:49,904 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.3220 sec 2024-11-15T00:50:49,906 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631849905"}]},"ts":"1731631849905"} 2024-11-15T00:50:49,908 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:50:49,908 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:50:49,911 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.3380 sec 2024-11-15T00:50:50,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=139 2024-11-15T00:50:50,717 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:50,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:50:50,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:50,719 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=143, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:50,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T00:50:50,725 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=143, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:50,727 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387 2024-11-15T00:50:50,731 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/recovered.edits] 2024-11-15T00:50:50,733 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/0277696bc0b947f39dd05ceebbe302b8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/0277696bc0b947f39dd05ceebbe302b8 2024-11-15T00:50:50,734 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/0547f2c62ed0484b85cdf6c4d99dbc68 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/0547f2c62ed0484b85cdf6c4d99dbc68 2024-11-15T00:50:50,735 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/ed747f7f812c44f1a60afd6a21dafe7b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/A/ed747f7f812c44f1a60afd6a21dafe7b 2024-11-15T00:50:50,736 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c20f8dd5a2884bfe8d3fded2652a7937 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/c20f8dd5a2884bfe8d3fded2652a7937 2024-11-15T00:50:50,737 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/e698c2afae74416c8b5358aaf73b7207 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/e698c2afae74416c8b5358aaf73b7207 2024-11-15T00:50:50,738 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/fb290ffd02394a07b7fde578d950a983 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/B/fb290ffd02394a07b7fde578d950a983 2024-11-15T00:50:50,740 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/44410571e1d045a8943224557815c621 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/44410571e1d045a8943224557815c621 2024-11-15T00:50:50,741 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7f6355d4edeb432f830f668508d4752a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/7f6355d4edeb432f830f668508d4752a 2024-11-15T00:50:50,742 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/ef3530a1297a41de873d6032089653d8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/C/ef3530a1297a41de873d6032089653d8 2024-11-15T00:50:50,744 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/recovered.edits/493.seqid to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387/recovered.edits/493.seqid 2024-11-15T00:50:50,745 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/0015d049952a9227751d617c5d05c387 2024-11-15T00:50:50,745 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:50:50,746 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=143, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:50,748 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:50:50,750 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:50:50,751 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=143, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:50,751 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:50:50,751 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631850751"}]},"ts":"9223372036854775807"} 2024-11-15T00:50:50,754 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:50:50,754 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 0015d049952a9227751d617c5d05c387, NAME => 'TestAcidGuarantees,,1731631817391.0015d049952a9227751d617c5d05c387.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:50:50,754 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:50:50,754 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631850754"}]},"ts":"9223372036854775807"} 2024-11-15T00:50:50,756 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:50:50,758 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=143, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:50,759 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-15T00:50:50,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=143 2024-11-15T00:50:50,827 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:50:50,827 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:50,837 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=236 (was 241), OpenFileDescriptor=443 (was 461), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=1001 (was 977) - SystemLoadAverage LEAK? -, ProcessCount=12 (was 11) - ProcessCount LEAK? -, AvailableMemoryMB=2871 (was 2667) - AvailableMemoryMB LEAK? - 2024-11-15T00:50:50,846 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=236, OpenFileDescriptor=443, MaxFileDescriptor=1048576, SystemLoadAverage=1001, ProcessCount=11, AvailableMemoryMB=2870 2024-11-15T00:50:50,847 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:50:50,847 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.3 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:50:50,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:50,849 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-15T00:50:50,849 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:50,850 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.3 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 144 2024-11-15T00:50:50,850 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-15T00:50:50,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:50:50,859 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742347_1523 (size=959) 2024-11-15T00:50:50,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:50:51,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:50:51,261 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd 2024-11-15T00:50:51,266 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742348_1524 (size=53) 2024-11-15T00:50:51,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:50:51,667 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:50:51,667 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 4c1752556c034b2ca5cdf7bf0db8cd94, disabling compactions & flushes 2024-11-15T00:50:51,667 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:51,667 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:51,667 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. after waiting 0 ms 2024-11-15T00:50:51,667 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:51,667 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:51,667 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 4c1752556c034b2ca5cdf7bf0db8cd94: Waiting for close lock at 1731631851667Disabling compacts and flushes for region at 1731631851667Disabling writes for close at 1731631851667Writing region close event to WAL at 1731631851667Closed at 1731631851667 2024-11-15T00:50:51,669 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-15T00:50:51,670 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1731631851669"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1731631851669"}]},"ts":"1731631851669"} 2024-11-15T00:50:51,673 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-15T00:50:51,675 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-15T00:50:51,675 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631851675"}]},"ts":"1731631851675"} 2024-11-15T00:50:51,687 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-15T00:50:51,688 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, ASSIGN}] 2024-11-15T00:50:51,689 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, ASSIGN 2024-11-15T00:50:51,689 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, ASSIGN; state=OFFLINE, location=fb2f6cb7846a,42925,1731631698709; forceNewPlan=false, retain=false 2024-11-15T00:50:51,840 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=4c1752556c034b2ca5cdf7bf0db8cd94, regionState=OPENING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:51,842 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=145, ppid=144, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, ASSIGN because future has completed 2024-11-15T00:50:51,843 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:50:51,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:50:51,999 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:51,999 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:50:52,000 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,000 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:50:52,000 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7794): checking encryption for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,000 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(7797): checking classloading for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,002 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,003 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c1752556c034b2ca5cdf7bf0db8cd94 columnFamilyName A 2024-11-15T00:50:52,004 DEBUG [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:52,005 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(327): Store=4c1752556c034b2ca5cdf7bf0db8cd94/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:50:52,005 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,007 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c1752556c034b2ca5cdf7bf0db8cd94 columnFamilyName B 2024-11-15T00:50:52,007 DEBUG [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:52,008 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(327): Store=4c1752556c034b2ca5cdf7bf0db8cd94/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:50:52,008 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,012 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c1752556c034b2ca5cdf7bf0db8cd94 columnFamilyName C 2024-11-15T00:50:52,012 DEBUG [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:52,012 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(327): Store=4c1752556c034b2ca5cdf7bf0db8cd94/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:50:52,012 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1038): replaying wal for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,013 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,013 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,014 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1048): stopping wal replay for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,014 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1060): Cleaning up temporary data for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,015 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:50:52,017 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1093): writing seq id for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,019 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-15T00:50:52,020 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1114): Opened 4c1752556c034b2ca5cdf7bf0db8cd94; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=74894170, jitterRate=0.11601009964942932}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:50:52,020 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:52,020 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegion(1006): Region open journal for 4c1752556c034b2ca5cdf7bf0db8cd94: Running coprocessor pre-open hook at 1731631852000Writing region info on filesystem at 1731631852000Initializing all the Stores at 1731631852001 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631852001Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631852001Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631852001Cleaning up temporary data from old regions at 1731631852014 (+13 ms)Running coprocessor post-open hooks at 1731631852020 (+6 ms)Region opened successfully at 1731631852020 2024-11-15T00:50:52,021 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., pid=146, masterSystemTime=1731631851996 2024-11-15T00:50:52,023 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:52,023 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=146}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:52,024 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=145 updating hbase:meta row=4c1752556c034b2ca5cdf7bf0db8cd94, regionState=OPEN, openSeqNum=2, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:52,027 WARN [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=46397 {}] assignment.AssignmentManager(1543): Unable to acquire lock for regionNode state=OPEN, location=fb2f6cb7846a,42925,1731631698709, table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94. It is likely that another thread is currently holding the lock. To avoid deadlock, skip execution for now. 2024-11-15T00:50:52,031 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=146, ppid=145, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:50:52,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-15T00:50:52,034 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 in 189 msec 2024-11-15T00:50:52,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=145, resume processing ppid=144 2024-11-15T00:50:52,040 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, ppid=144, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, ASSIGN in 346 msec 2024-11-15T00:50:52,042 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-15T00:50:52,043 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631852042"}]},"ts":"1731631852042"} 2024-11-15T00:50:52,045 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-15T00:50:52,046 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=144, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-15T00:50:52,049 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1990 sec 2024-11-15T00:50:53,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=144 2024-11-15T00:50:53,007 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:53,008 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75b3b1fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,008 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,008 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,009 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@251011cf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,011 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,012 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55790, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,013 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e092f43, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,014 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:53,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:53,017 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33578, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:53,018 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-15T00:50:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is fb2f6cb7846a,46397,1731631697729 2024-11-15T00:50:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@68cbdbfd 2024-11-15T00:50:53,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-15T00:50:53,020 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55804, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-15T00:50:53,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-15T00:50:53,023 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.3 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-15T00:50:53,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=147, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-15T00:50:53,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742349_1525 (size=995) 2024-11-15T00:50:53,438 DEBUG [PEWorker-1 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-15T00:50:53,438 INFO [PEWorker-1 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-15T00:50:53,440 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:50:53,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, REOPEN/MOVE}] 2024-11-15T00:50:53,445 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, REOPEN/MOVE 2024-11-15T00:50:53,446 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=4c1752556c034b2ca5cdf7bf0db8cd94, regionState=CLOSING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:53,448 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, REOPEN/MOVE because future has completed 2024-11-15T00:50:53,448 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:50:53,448 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:50:53,601 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(122): Close 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,601 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:50:53,601 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1722): Closing 4c1752556c034b2ca5cdf7bf0db8cd94, disabling compactions & flushes 2024-11-15T00:50:53,601 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:53,601 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:53,601 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. after waiting 0 ms 2024-11-15T00:50:53,601 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:53,605 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-15T00:50:53,605 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:53,605 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegion(1676): Region close journal for 4c1752556c034b2ca5cdf7bf0db8cd94: Waiting for close lock at 1731631853601Running coprocessor pre-close hooks at 1731631853601Disabling compacts and flushes for region at 1731631853601Disabling writes for close at 1731631853601Writing region close event to WAL at 1731631853602 (+1 ms)Running coprocessor post-close hooks at 1731631853605 (+3 ms)Closed at 1731631853605 2024-11-15T00:50:53,605 WARN [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] regionserver.HRegionServer(3299): Not adding moved region record: 4c1752556c034b2ca5cdf7bf0db8cd94 to self. 2024-11-15T00:50:53,607 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=150}] handler.UnassignRegionHandler(157): Closed 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,607 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=4c1752556c034b2ca5cdf7bf0db8cd94, regionState=CLOSED 2024-11-15T00:50:53,610 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=150, ppid=149, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:50:53,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=150, resume processing ppid=149 2024-11-15T00:50:53,612 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, ppid=149, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 in 162 msec 2024-11-15T00:50:53,613 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, REOPEN/MOVE; state=CLOSED, location=fb2f6cb7846a,42925,1731631698709; forceNewPlan=false, retain=true 2024-11-15T00:50:53,766 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=4c1752556c034b2ca5cdf7bf0db8cd94, regionState=OPENING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:53,767 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=148, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, REOPEN/MOVE because future has completed 2024-11-15T00:50:53,767 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:50:53,922 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:53,922 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7752): Opening region: {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} 2024-11-15T00:50:53,922 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,922 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-15T00:50:53,922 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7794): checking encryption for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,922 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(7797): checking classloading for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,928 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,929 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c1752556c034b2ca5cdf7bf0db8cd94 columnFamilyName A 2024-11-15T00:50:53,930 DEBUG [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:53,930 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(327): Store=4c1752556c034b2ca5cdf7bf0db8cd94/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:50:53,930 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,931 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c1752556c034b2ca5cdf7bf0db8cd94 columnFamilyName B 2024-11-15T00:50:53,931 DEBUG [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:53,931 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(327): Store=4c1752556c034b2ca5cdf7bf0db8cd94/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:50:53,931 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,932 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 4c1752556c034b2ca5cdf7bf0db8cd94 columnFamilyName C 2024-11-15T00:50:53,932 DEBUG [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:53,932 INFO [StoreOpener-4c1752556c034b2ca5cdf7bf0db8cd94-1 {}] regionserver.HStore(327): Store=4c1752556c034b2ca5cdf7bf0db8cd94/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-15T00:50:53,932 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1038): replaying wal for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,933 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,934 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,934 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1048): stopping wal replay for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,934 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1060): Cleaning up temporary data for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,935 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-15T00:50:53,936 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1093): writing seq id for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,937 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1114): Opened 4c1752556c034b2ca5cdf7bf0db8cd94; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=71104852, jitterRate=0.05954486131668091}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-15T00:50:53,937 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:53,937 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegion(1006): Region open journal for 4c1752556c034b2ca5cdf7bf0db8cd94: Running coprocessor pre-open hook at 1731631853922Writing region info on filesystem at 1731631853922Initializing all the Stores at 1731631853923 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631853923Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631853927 (+4 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1731631853927Cleaning up temporary data from old regions at 1731631853934 (+7 ms)Running coprocessor post-open hooks at 1731631853937 (+3 ms)Region opened successfully at 1731631853937 2024-11-15T00:50:53,938 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., pid=151, masterSystemTime=1731631853919 2024-11-15T00:50:53,940 DEBUG [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:53,940 INFO [RS_OPEN_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_OPEN_REGION, pid=151}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:53,940 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=149 updating hbase:meta row=4c1752556c034b2ca5cdf7bf0db8cd94, regionState=OPEN, openSeqNum=5, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:53,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=151, ppid=149, state=RUNNABLE, hasLock=false; OpenRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:50:53,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=149 2024-11-15T00:50:53,947 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=149, state=SUCCESS, hasLock=false; OpenRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 in 177 msec 2024-11-15T00:50:53,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=148 2024-11-15T00:50:53,949 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=148, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, REOPEN/MOVE in 503 msec 2024-11-15T00:50:53,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-15T00:50:53,951 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 510 msec 2024-11-15T00:50:53,953 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 929 msec 2024-11-15T00:50:53,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=147 2024-11-15T00:50:53,954 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:53,955 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22842932, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,955 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,955 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,956 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74fbe3d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,956 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,957 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,958 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55812, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5ba0cacd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,959 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c5fa63a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,959 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,959 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,960 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@70008fc3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,960 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,961 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,961 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55824, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,962 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@45f7d475, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,962 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@71a933d5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,962 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,963 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,963 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,963 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@369b3268, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,964 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,965 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55848, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,965 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36f3836a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,968 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66f73499, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,969 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,969 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,969 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@285fcca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,970 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,971 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55856, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b0ed526, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,972 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f42478c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,972 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,972 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,973 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,973 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@61ed85ed, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,974 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,975 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55868, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,975 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bb43784, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,976 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6f6185aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,976 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,976 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,977 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2146545d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,977 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,978 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55892, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,979 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@564442b5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,979 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@476ca966, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,979 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,980 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,981 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4c6e1417, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,981 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,982 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,983 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55896, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,984 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@492341ff, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,986 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@78b91db9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,986 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,986 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,987 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688e7289, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,987 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,988 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,989 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55918, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,991 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ba1d9ab, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,992 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60dcf631, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,992 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,992 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,993 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c28402, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,993 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,994 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,995 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55926, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,995 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@16b314f9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,996 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6b3d9872, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,996 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request fb2f6cb7846a,46397,-1 for getting cluster id 2024-11-15T00:50:53,996 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-15T00:50:53,997 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5' 2024-11-15T00:50:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-15T00:50:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "4ff0fc1d-3804-41ef-bcba-8a3ba6d0fdc5" 2024-11-15T00:50:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3898962d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:53,997 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [fb2f6cb7846a,46397,-1] 2024-11-15T00:50:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-15T00:50:53,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:50:53,999 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:55936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-15T00:50:53,999 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3cd6937c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-15T00:50:54,000 DEBUG [Thread-2356 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:54,001 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:54,002 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33592, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:54,004 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:50:54,004 DEBUG [Thread-2352 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:54,005 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:54,006 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:54,008 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:54,008 DEBUG [Thread-2359 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:54,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:50:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:54,012 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:54,014 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:54,016 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:50:54,017 DEBUG [Thread-2350 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:54,017 DEBUG [Thread-2361 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:54,018 DEBUG [Thread-2350 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:54,018 DEBUG [Thread-2350 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:54,018 DEBUG [Thread-2354 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:54,019 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33614, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:54,019 DEBUG [Thread-2361 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:54,020 DEBUG [Thread-2361 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:54,020 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:54,020 DEBUG [Thread-2365 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:54,020 DEBUG [Thread-2363 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:54,020 DEBUG [Thread-2348 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:54,021 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:54,021 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33616, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:54,022 DEBUG [Thread-2367 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-15T00:50:54,022 DEBUG [Thread-2367 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:54,023 DEBUG [Thread-2367 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:54,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:54,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:54,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=fb2f6cb7846a,42925,1731631698709, seqNum=-1] 2024-11-15T00:50:54,024 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-15T00:50:54,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:54,024 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:54,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33626, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:54,025 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:50:54,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33658, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:54,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33674, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:54,025 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33642, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:54,025 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:50:54,026 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.3:33682, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-15T00:50:54,028 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:50:54,028 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:50:54,029 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:50:54,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:50:54,031 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:50:54,032 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5] 2024-11-15T00:50:54,033 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:54,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-15T00:50:54,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T00:50:54,045 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:54,047 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:54,048 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:54,070 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111543512be5fb034459b1536e9077266406_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631854023/Put/seqid=0 2024-11-15T00:50:54,085 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631914078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,087 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,087 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631914081, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631914083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,088 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,092 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631914086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,094 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 5 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631914084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,100 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742350_1526 (size=12154) 2024-11-15T00:50:54,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T00:50:54,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631914196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,199 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631914197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,200 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631914197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,200 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,201 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T00:50:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,201 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631914208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631914209, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,212 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,213 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,354 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T00:50:54,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:54,354 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,354 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T00:50:54,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631914407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631914407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631914407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631914417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,419 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631914417, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,419 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,507 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T00:50:54,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:54,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,508 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,517 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:54,521 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111543512be5fb034459b1536e9077266406_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111543512be5fb034459b1536e9077266406_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:54,522 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/bb25094849a940f4aada0393fbd0bd73, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:50:54,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/bb25094849a940f4aada0393fbd0bd73 is 175, key is test_row_0/A:col10/1731631854023/Put/seqid=0 2024-11-15T00:50:54,529 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742351_1527 (size=30955) 2024-11-15T00:50:54,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T00:50:54,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:54,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T00:50:54,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631914716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631914716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,719 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631914716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,719 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,729 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631914727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,730 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631914727, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:54,730 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:54,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T00:50:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,930 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/bb25094849a940f4aada0393fbd0bd73 2024-11-15T00:50:54,964 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/7f14a1fc575a4eb2af9dcb878c5e94e1 is 50, key is test_row_0/B:col10/1731631854023/Put/seqid=0 2024-11-15T00:50:54,973 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742352_1528 (size=12001) 2024-11-15T00:50:54,974 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/7f14a1fc575a4eb2af9dcb878c5e94e1 2024-11-15T00:50:54,976 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T00:50:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:54,976 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:54,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:55,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/cbe146b45046416384af802e506fe4f8 is 50, key is test_row_0/C:col10/1731631854023/Put/seqid=0 2024-11-15T00:50:55,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742353_1529 (size=12001) 2024-11-15T00:50:55,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T00:50:55,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:55,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:55,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:55,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:55,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:55,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:55,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T00:50:55,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:55,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631915227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:55,227 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:55,228 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:55,228 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631915227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:55,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:55,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631915227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:55,229 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:55,239 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631915237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:55,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631915237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:55,240 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:55,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T00:50:55,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:55,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:55,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:55,287 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:55,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:55,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:55,411 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-15T00:50:55,417 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/cbe146b45046416384af802e506fe4f8 2024-11-15T00:50:55,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T00:50:55,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:55,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:55,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:55,441 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] handler.RSProcedureHandler(58): pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:55,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=153 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:55,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=153 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:55,455 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/bb25094849a940f4aada0393fbd0bd73 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/bb25094849a940f4aada0393fbd0bd73 2024-11-15T00:50:55,459 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/bb25094849a940f4aada0393fbd0bd73, entries=150, sequenceid=15, filesize=30.2 K 2024-11-15T00:50:55,461 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/7f14a1fc575a4eb2af9dcb878c5e94e1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/7f14a1fc575a4eb2af9dcb878c5e94e1 2024-11-15T00:50:55,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/7f14a1fc575a4eb2af9dcb878c5e94e1, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T00:50:55,471 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/cbe146b45046416384af802e506fe4f8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/cbe146b45046416384af802e506fe4f8 2024-11-15T00:50:55,475 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/cbe146b45046416384af802e506fe4f8, entries=150, sequenceid=15, filesize=11.7 K 2024-11-15T00:50:55,478 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1454ms, sequenceid=15, compaction requested=false 2024-11-15T00:50:55,479 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-15T00:50:55,482 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:50:55,595 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-15T00:50:55,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:55,596 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:50:55,606 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c9b9f68d77064774b5533d5f3c35eec0_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631854084/Put/seqid=0 2024-11-15T00:50:55,639 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742354_1530 (size=12154) 2024-11-15T00:50:55,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:55,655 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c9b9f68d77064774b5533d5f3c35eec0_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c9b9f68d77064774b5533d5f3c35eec0_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:55,656 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/f19d2438629543918a181e51377a7f7d, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:50:55,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/f19d2438629543918a181e51377a7f7d is 175, key is test_row_0/A:col10/1731631854084/Put/seqid=0 2024-11-15T00:50:55,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742355_1531 (size=30955) 2024-11-15T00:50:55,685 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/f19d2438629543918a181e51377a7f7d 2024-11-15T00:50:55,700 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/1e6aef98e7624870ab6a9dee0e711acf is 50, key is test_row_0/B:col10/1731631854084/Put/seqid=0 2024-11-15T00:50:55,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742356_1532 (size=12001) 2024-11-15T00:50:56,158 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/1e6aef98e7624870ab6a9dee0e711acf 2024-11-15T00:50:56,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/2b622549d1cc429d93e6b6298ad592e1 is 50, key is test_row_0/C:col10/1731631854084/Put/seqid=0 2024-11-15T00:50:56,171 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742357_1533 (size=12001) 2024-11-15T00:50:56,172 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/2b622549d1cc429d93e6b6298ad592e1 2024-11-15T00:50:56,176 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/f19d2438629543918a181e51377a7f7d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f19d2438629543918a181e51377a7f7d 2024-11-15T00:50:56,183 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f19d2438629543918a181e51377a7f7d, entries=150, sequenceid=40, filesize=30.2 K 2024-11-15T00:50:56,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/1e6aef98e7624870ab6a9dee0e711acf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/1e6aef98e7624870ab6a9dee0e711acf 2024-11-15T00:50:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,190 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/1e6aef98e7624870ab6a9dee0e711acf, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T00:50:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/2b622549d1cc429d93e6b6298ad592e1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2b622549d1cc429d93e6b6298ad592e1 2024-11-15T00:50:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,195 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2b622549d1cc429d93e6b6298ad592e1, entries=150, sequenceid=40, filesize=11.7 K 2024-11-15T00:50:56,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,196 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=0 B/0 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 600ms, sequenceid=40, compaction requested=false 2024-11-15T00:50:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:50:56,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:56,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-15T00:50:56,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T00:50:56,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-15T00:50:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-15T00:50:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,201 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1510 sec 2024-11-15T00:50:56,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,203 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 2.1680 sec 2024-11-15T00:50:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:56,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,288 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:56,288 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:50:56,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,301 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e6780a8b72254f849e901ec51b455888_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631856278/Put/seqid=0 2024-11-15T00:50:56,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,336 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631916330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631916331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631916331, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,337 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631916333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,338 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631916334, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,339 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,345 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742359_1535 (size=24358) 2024-11-15T00:50:56,346 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:56,351 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115e6780a8b72254f849e901ec51b455888_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e6780a8b72254f849e901ec51b455888_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:56,354 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/b388374d17964cc7a5d294230f5889a3, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:50:56,354 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/b388374d17964cc7a5d294230f5889a3 is 175, key is test_row_0/A:col10/1731631856278/Put/seqid=0 2024-11-15T00:50:56,356 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742358_1534 (size=73994) 2024-11-15T00:50:56,356 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/b388374d17964cc7a5d294230f5889a3 2024-11-15T00:50:56,366 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/0d384dbb695e412ebf4e36a187d8bdd9 is 50, key is test_row_0/B:col10/1731631856278/Put/seqid=0 2024-11-15T00:50:56,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742360_1536 (size=12001) 2024-11-15T00:50:56,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631916446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631916447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631916447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,448 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,448 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631916447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,449 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631916447, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,449 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,660 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631916657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631916657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,660 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,660 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631916658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631916658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,661 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,661 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631916658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,662 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/0d384dbb695e412ebf4e36a187d8bdd9 2024-11-15T00:50:56,812 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/0f7d60f8e2954498834beb9251cc1da9 is 50, key is test_row_0/C:col10/1731631856278/Put/seqid=0 2024-11-15T00:50:56,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742361_1537 (size=12001) 2024-11-15T00:50:56,969 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631916967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631916967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,970 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631916967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631916967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,971 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:56,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631916967, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:56,971 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,238 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/0f7d60f8e2954498834beb9251cc1da9 2024-11-15T00:50:57,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/b388374d17964cc7a5d294230f5889a3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/b388374d17964cc7a5d294230f5889a3 2024-11-15T00:50:57,244 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/b388374d17964cc7a5d294230f5889a3, entries=400, sequenceid=52, filesize=72.3 K 2024-11-15T00:50:57,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/0d384dbb695e412ebf4e36a187d8bdd9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/0d384dbb695e412ebf4e36a187d8bdd9 2024-11-15T00:50:57,250 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/0d384dbb695e412ebf4e36a187d8bdd9, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T00:50:57,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/0f7d60f8e2954498834beb9251cc1da9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/0f7d60f8e2954498834beb9251cc1da9 2024-11-15T00:50:57,258 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/0f7d60f8e2954498834beb9251cc1da9, entries=150, sequenceid=52, filesize=11.7 K 2024-11-15T00:50:57,258 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 970ms, sequenceid=52, compaction requested=true 2024-11-15T00:50:57,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:50:57,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:50:57,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:57,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:50:57,259 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:57,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:57,259 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:57,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:50:57,259 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:57,259 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:57,259 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/B is initiating minor compaction (all files) 2024-11-15T00:50:57,259 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/B in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:57,260 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/7f14a1fc575a4eb2af9dcb878c5e94e1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/1e6aef98e7624870ab6a9dee0e711acf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/0d384dbb695e412ebf4e36a187d8bdd9] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=35.2 K 2024-11-15T00:50:57,260 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 135904 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:57,260 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/A is initiating minor compaction (all files) 2024-11-15T00:50:57,260 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/A in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:57,260 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 7f14a1fc575a4eb2af9dcb878c5e94e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731631854013 2024-11-15T00:50:57,260 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/bb25094849a940f4aada0393fbd0bd73, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f19d2438629543918a181e51377a7f7d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/b388374d17964cc7a5d294230f5889a3] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=132.7 K 2024-11-15T00:50:57,260 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:57,260 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/bb25094849a940f4aada0393fbd0bd73, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f19d2438629543918a181e51377a7f7d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/b388374d17964cc7a5d294230f5889a3] 2024-11-15T00:50:57,260 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1e6aef98e7624870ab6a9dee0e711acf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731631854074 2024-11-15T00:50:57,260 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting bb25094849a940f4aada0393fbd0bd73, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731631854013 2024-11-15T00:50:57,261 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0d384dbb695e412ebf4e36a187d8bdd9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731631856278 2024-11-15T00:50:57,261 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f19d2438629543918a181e51377a7f7d, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731631854074 2024-11-15T00:50:57,261 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting b388374d17964cc7a5d294230f5889a3, keycount=400, bloomtype=ROW, size=72.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731631856250 2024-11-15T00:50:57,288 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#B#compaction#454 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:57,288 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/ce66d56c72674cb791b753d302b58bca is 50, key is test_row_0/B:col10/1731631856278/Put/seqid=0 2024-11-15T00:50:57,293 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:50:57,301 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115ff3caf7fd96e4b06932d000a0fded66c_4c1752556c034b2ca5cdf7bf0db8cd94 store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:50:57,304 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115ff3caf7fd96e4b06932d000a0fded66c_4c1752556c034b2ca5cdf7bf0db8cd94, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:50:57,305 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ff3caf7fd96e4b06932d000a0fded66c_4c1752556c034b2ca5cdf7bf0db8cd94 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:50:57,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742362_1538 (size=12104) 2024-11-15T00:50:57,322 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/ce66d56c72674cb791b753d302b58bca as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ce66d56c72674cb791b753d302b58bca 2024-11-15T00:50:57,328 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/B of 4c1752556c034b2ca5cdf7bf0db8cd94 into ce66d56c72674cb791b753d302b58bca(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:57,329 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:50:57,329 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/B, priority=13, startTime=1731631857259; duration=0sec 2024-11-15T00:50:57,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742363_1539 (size=4469) 2024-11-15T00:50:57,329 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:50:57,329 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:B 2024-11-15T00:50:57,329 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:50:57,331 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:50:57,331 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/C is initiating minor compaction (all files) 2024-11-15T00:50:57,331 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/C in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:57,331 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/cbe146b45046416384af802e506fe4f8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2b622549d1cc429d93e6b6298ad592e1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/0f7d60f8e2954498834beb9251cc1da9] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=35.2 K 2024-11-15T00:50:57,331 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#A#compaction#455 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:57,332 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/48eaa8142e1242489e25f24eca5ee9d8 is 175, key is test_row_0/A:col10/1731631856278/Put/seqid=0 2024-11-15T00:50:57,332 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting cbe146b45046416384af802e506fe4f8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1731631854013 2024-11-15T00:50:57,333 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b622549d1cc429d93e6b6298ad592e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1731631854074 2024-11-15T00:50:57,334 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f7d60f8e2954498834beb9251cc1da9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731631856278 2024-11-15T00:50:57,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742364_1540 (size=31058) 2024-11-15T00:50:57,341 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#C#compaction#456 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:50:57,342 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/4beb458d261c4a6198877f0b03738ae8 is 50, key is test_row_0/C:col10/1731631856278/Put/seqid=0 2024-11-15T00:50:57,357 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/48eaa8142e1242489e25f24eca5ee9d8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/48eaa8142e1242489e25f24eca5ee9d8 2024-11-15T00:50:57,364 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/A of 4c1752556c034b2ca5cdf7bf0db8cd94 into 48eaa8142e1242489e25f24eca5ee9d8(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:57,364 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:50:57,364 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/A, priority=13, startTime=1731631857258; duration=0sec 2024-11-15T00:50:57,364 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:57,364 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:A 2024-11-15T00:50:57,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742365_1541 (size=12104) 2024-11-15T00:50:57,373 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/4beb458d261c4a6198877f0b03738ae8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4beb458d261c4a6198877f0b03738ae8 2024-11-15T00:50:57,378 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/C of 4c1752556c034b2ca5cdf7bf0db8cd94 into 4beb458d261c4a6198877f0b03738ae8(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:50:57,378 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:50:57,378 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/C, priority=13, startTime=1731631857259; duration=0sec 2024-11-15T00:50:57,378 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:50:57,378 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:C 2024-11-15T00:50:57,481 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:50:57,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:50:57,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:57,497 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115312f9f09152144d89c499a2cde3726a5_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631857479/Put/seqid=0 2024-11-15T00:50:57,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631917495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631917492, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631917495, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,501 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631917497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631917497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,502 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,514 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742366_1542 (size=12154) 2024-11-15T00:50:57,515 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:57,520 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115312f9f09152144d89c499a2cde3726a5_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115312f9f09152144d89c499a2cde3726a5_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:57,523 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/70c1c743ea5448d2a5e47d6e5e7247ac, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:50:57,523 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/70c1c743ea5448d2a5e47d6e5e7247ac is 175, key is test_row_0/A:col10/1731631857479/Put/seqid=0 2024-11-15T00:50:57,550 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742367_1543 (size=30955) 2024-11-15T00:50:57,550 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=81, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/70c1c743ea5448d2a5e47d6e5e7247ac 2024-11-15T00:50:57,558 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/56b11721848b46ba9e3c0a67cd03c523 is 50, key is test_row_0/B:col10/1731631857479/Put/seqid=0 2024-11-15T00:50:57,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742368_1544 (size=12001) 2024-11-15T00:50:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631917607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631917607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631917607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631917607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,614 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631917612, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,615 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631917817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631917817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631917817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631917817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:57,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631917817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:57,820 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:57,978 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/56b11721848b46ba9e3c0a67cd03c523 2024-11-15T00:50:57,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/86683b2c27834ba59cf460522f629adc is 50, key is test_row_0/C:col10/1731631857479/Put/seqid=0 2024-11-15T00:50:58,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742369_1545 (size=12001) 2024-11-15T00:50:58,128 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631918127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,129 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631918127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,130 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631918127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631918127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,133 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,136 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631918133, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,140 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-15T00:50:58,229 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:50:58,232 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:50:58,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-15T00:50:58,235 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:50:58,237 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:50:58,237 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:50:58,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T00:50:58,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T00:50:58,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T00:50:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:58,391 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:58,391 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:58,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:58,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:50:58,451 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=81 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/86683b2c27834ba59cf460522f629adc 2024-11-15T00:50:58,459 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/70c1c743ea5448d2a5e47d6e5e7247ac as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/70c1c743ea5448d2a5e47d6e5e7247ac 2024-11-15T00:50:58,465 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/70c1c743ea5448d2a5e47d6e5e7247ac, entries=150, sequenceid=81, filesize=30.2 K 2024-11-15T00:50:58,466 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/56b11721848b46ba9e3c0a67cd03c523 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/56b11721848b46ba9e3c0a67cd03c523 2024-11-15T00:50:58,470 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/56b11721848b46ba9e3c0a67cd03c523, entries=150, sequenceid=81, filesize=11.7 K 2024-11-15T00:50:58,470 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/86683b2c27834ba59cf460522f629adc as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/86683b2c27834ba59cf460522f629adc 2024-11-15T00:50:58,474 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/86683b2c27834ba59cf460522f629adc, entries=150, sequenceid=81, filesize=11.7 K 2024-11-15T00:50:58,475 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 994ms, sequenceid=81, compaction requested=false 2024-11-15T00:50:58,475 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:50:58,546 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-15T00:50:58,546 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:50:58,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:50:58,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T00:50:58,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115781e25b4fc874ece8b7cba80f347e493_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631857493/Put/seqid=0 2024-11-15T00:50:58,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742370_1546 (size=12154) 2024-11-15T00:50:58,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:58,639 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:50:58,675 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631918669, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631918670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,676 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631918670, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,676 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,677 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631918676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,680 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,681 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631918678, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,681 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,682 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,788 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631918787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631918787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,789 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631918787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631918790, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,794 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631918791, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:58,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:58,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T00:50:58,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:50:58,973 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115781e25b4fc874ece8b7cba80f347e493_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115781e25b4fc874ece8b7cba80f347e493_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:50:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/29ce6c68d5ae4e00a31e78fc93d0327a, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:50:58,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/29ce6c68d5ae4e00a31e78fc93d0327a is 175, key is test_row_0/A:col10/1731631857493/Put/seqid=0 2024-11-15T00:50:58,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631918998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631918998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:58,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:58,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631918999, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742371_1547 (size=30955) 2024-11-15T00:50:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,000 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631919007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,009 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631919007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,010 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631919306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,311 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631919307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,312 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631919307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,312 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,313 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,322 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631919317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631919317, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T00:50:59,401 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=92, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/29ce6c68d5ae4e00a31e78fc93d0327a 2024-11-15T00:50:59,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/5cb131d640f844ba913cda004d30744d is 50, key is test_row_0/B:col10/1731631857493/Put/seqid=0 2024-11-15T00:50:59,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742372_1548 (size=12001) 2024-11-15T00:50:59,821 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631919818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631919818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,822 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631919818, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,822 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,823 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,845 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/5cb131d640f844ba913cda004d30744d 2024-11-15T00:50:59,846 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631919839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,847 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:50:59,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631919839, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:50:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:50:59,847 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:50:59,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/f6dd7317f3b54d4c846e86f4219fc6ab is 50, key is test_row_0/C:col10/1731631857493/Put/seqid=0 2024-11-15T00:50:59,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742373_1549 (size=12001) 2024-11-15T00:51:00,270 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=92 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/f6dd7317f3b54d4c846e86f4219fc6ab 2024-11-15T00:51:00,275 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/29ce6c68d5ae4e00a31e78fc93d0327a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/29ce6c68d5ae4e00a31e78fc93d0327a 2024-11-15T00:51:00,279 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/29ce6c68d5ae4e00a31e78fc93d0327a, entries=150, sequenceid=92, filesize=30.2 K 2024-11-15T00:51:00,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/5cb131d640f844ba913cda004d30744d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/5cb131d640f844ba913cda004d30744d 2024-11-15T00:51:00,284 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/5cb131d640f844ba913cda004d30744d, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T00:51:00,285 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/f6dd7317f3b54d4c846e86f4219fc6ab as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f6dd7317f3b54d4c846e86f4219fc6ab 2024-11-15T00:51:00,290 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f6dd7317f3b54d4c846e86f4219fc6ab, entries=150, sequenceid=92, filesize=11.7 K 2024-11-15T00:51:00,295 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1749ms, sequenceid=92, compaction requested=true 2024-11-15T00:51:00,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:00,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:00,295 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-15T00:51:00,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-15T00:51:00,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-15T00:51:00,298 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0600 sec 2024-11-15T00:51:00,300 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.0670 sec 2024-11-15T00:51:00,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-15T00:51:00,408 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:51:00,408 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:51:00,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-15T00:51:00,412 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:51:00,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T00:51:00,416 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:51:00,417 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:51:00,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T00:51:00,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-15T00:51:00,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:00,577 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:51:00,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115dadacaa2306b41e69ec5e1377e0d8f40_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631858667/Put/seqid=0 2024-11-15T00:51:00,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742374_1550 (size=12154) 2024-11-15T00:51:00,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:00,642 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115dadacaa2306b41e69ec5e1377e0d8f40_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dadacaa2306b41e69ec5e1377e0d8f40_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:00,644 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/499d2dc1c4b84a988e271f00c3c4160e, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:00,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/499d2dc1c4b84a988e271f00c3c4160e is 175, key is test_row_0/A:col10/1731631858667/Put/seqid=0 2024-11-15T00:51:00,684 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742375_1551 (size=30955) 2024-11-15T00:51:00,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T00:51:00,833 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:00,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:00,859 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:00,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631920855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:00,862 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:00,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631920856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:00,864 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631920857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:00,865 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:00,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631920858, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,865 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,866 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:00,873 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 4c1752556c034b2ca5cdf7bf0db8cd94 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2202 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:51:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,873 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:00,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:00,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631920872, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:00,878 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 4c1752556c034b2ca5cdf7bf0db8cd94 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2203 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:51:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,879 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:00,973 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:00,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631920969, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,974 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:00,980 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:00,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631920978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:00,981 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:00,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631920978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:00,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:00,982 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T00:51:01,085 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/499d2dc1c4b84a988e271f00c3c4160e 2024-11-15T00:51:01,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/627b05101ece44a6be5e5f582e1a008e is 50, key is test_row_0/B:col10/1731631858667/Put/seqid=0 2024-11-15T00:51:01,142 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742376_1552 (size=12001) 2024-11-15T00:51:01,149 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/627b05101ece44a6be5e5f582e1a008e 2024-11-15T00:51:01,166 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/a318135540bf4b00a9358d0c2280b8c5 is 50, key is test_row_0/C:col10/1731631858667/Put/seqid=0 2024-11-15T00:51:01,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631921177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,190 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742377_1553 (size=12001) 2024-11-15T00:51:01,192 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631921187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,192 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,193 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631921187, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,193 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,204 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/a318135540bf4b00a9358d0c2280b8c5 2024-11-15T00:51:01,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/499d2dc1c4b84a988e271f00c3c4160e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/499d2dc1c4b84a988e271f00c3c4160e 2024-11-15T00:51:01,238 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/499d2dc1c4b84a988e271f00c3c4160e, entries=150, sequenceid=117, filesize=30.2 K 2024-11-15T00:51:01,238 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/627b05101ece44a6be5e5f582e1a008e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/627b05101ece44a6be5e5f582e1a008e 2024-11-15T00:51:01,242 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/627b05101ece44a6be5e5f582e1a008e, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T00:51:01,243 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/a318135540bf4b00a9358d0c2280b8c5 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a318135540bf4b00a9358d0c2280b8c5 2024-11-15T00:51:01,249 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a318135540bf4b00a9358d0c2280b8c5, entries=150, sequenceid=117, filesize=11.7 K 2024-11-15T00:51:01,250 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 673ms, sequenceid=117, compaction requested=true 2024-11-15T00:51:01,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:01,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:01,250 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-15T00:51:01,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-15T00:51:01,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-15T00:51:01,253 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 834 msec 2024-11-15T00:51:01,257 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 846 msec 2024-11-15T00:51:01,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:01,491 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:51:01,491 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T00:51:01,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c1aecd4d789948cbb0d19ca630506c5d_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631860852/Put/seqid=0 2024-11-15T00:51:01,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631921535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,544 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631921539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-15T00:51:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,547 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,547 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:51:01,548 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:51:01,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742378_1554 (size=12204) 2024-11-15T00:51:01,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-15T00:51:01,549 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:01,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T00:51:01,550 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:51:01,551 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:51:01,551 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:51:01,553 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631921547, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,554 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c1aecd4d789948cbb0d19ca630506c5d_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c1aecd4d789948cbb0d19ca630506c5d_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,555 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,555 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/c41d71974c7e43778f9226e5ea2a5e7d, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:01,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/c41d71974c7e43778f9226e5ea2a5e7d is 175, key is test_row_0/A:col10/1731631860852/Put/seqid=0 2024-11-15T00:51:01,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742379_1555 (size=31005) 2024-11-15T00:51:01,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T00:51:01,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631921659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,663 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631921659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,663 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,664 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631921659, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,664 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,703 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T00:51:01,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:01,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:01,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:01,703 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:01,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:01,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:01,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T00:51:01,862 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:01,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:01,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:01,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:01,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T00:51:01,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631921867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,870 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631921867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,871 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631921867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:01,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:02,006 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=129, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/c41d71974c7e43778f9226e5ea2a5e7d 2024-11-15T00:51:02,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/53d39b4fcc634d37b146c48b780ee9d0 is 50, key is test_row_0/B:col10/1731631860852/Put/seqid=0 2024-11-15T00:51:02,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T00:51:02,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:02,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,017 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742380_1556 (size=12051) 2024-11-15T00:51:02,173 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T00:51:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,173 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,173 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T00:51:02,181 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631922178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:02,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631922178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:02,182 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631922178, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,182 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:02,326 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T00:51:02,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:02,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,326 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,425 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/53d39b4fcc634d37b146c48b780ee9d0 2024-11-15T00:51:02,433 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/dd14a784626d4497943f470fbeb42dce is 50, key is test_row_0/C:col10/1731631860852/Put/seqid=0 2024-11-15T00:51:02,436 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742381_1557 (size=12051) 2024-11-15T00:51:02,480 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T00:51:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,480 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:02,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,481 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T00:51:02,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T00:51:02,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631922687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:02,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631922687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,691 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:02,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:02,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631922697, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,701 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:02,787 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T00:51:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,788 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,788 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:02,841 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=129 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/dd14a784626d4497943f470fbeb42dce 2024-11-15T00:51:02,862 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/c41d71974c7e43778f9226e5ea2a5e7d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/c41d71974c7e43778f9226e5ea2a5e7d 2024-11-15T00:51:02,872 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/c41d71974c7e43778f9226e5ea2a5e7d, entries=150, sequenceid=129, filesize=30.3 K 2024-11-15T00:51:02,873 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/53d39b4fcc634d37b146c48b780ee9d0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/53d39b4fcc634d37b146c48b780ee9d0 2024-11-15T00:51:02,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/53d39b4fcc634d37b146c48b780ee9d0, entries=150, sequenceid=129, filesize=11.8 K 2024-11-15T00:51:02,889 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/dd14a784626d4497943f470fbeb42dce as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/dd14a784626d4497943f470fbeb42dce 2024-11-15T00:51:02,892 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:02,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631922887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:02,893 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 4c1752556c034b2ca5cdf7bf0db8cd94 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4217 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:51:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,894 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:02,901 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/dd14a784626d4497943f470fbeb42dce, entries=150, sequenceid=129, filesize=11.8 K 2024-11-15T00:51:02,901 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:02,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631922898, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:02,902 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 4c1752556c034b2ca5cdf7bf0db8cd94 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4232 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:51:02,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:02,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:02,903 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1412ms, sequenceid=129, compaction requested=true 2024-11-15T00:51:02,903 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:51:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:51:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:51:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:51:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:51:02,904 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T00:51:02,904 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:51:02,904 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:51:02,906 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 154928 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:51:02,906 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/A is initiating minor compaction (all files) 2024-11-15T00:51:02,906 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/A in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,907 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/48eaa8142e1242489e25f24eca5ee9d8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/70c1c743ea5448d2a5e47d6e5e7247ac, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/29ce6c68d5ae4e00a31e78fc93d0327a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/499d2dc1c4b84a988e271f00c3c4160e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/c41d71974c7e43778f9226e5ea2a5e7d] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=151.3 K 2024-11-15T00:51:02,907 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,907 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/48eaa8142e1242489e25f24eca5ee9d8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/70c1c743ea5448d2a5e47d6e5e7247ac, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/29ce6c68d5ae4e00a31e78fc93d0327a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/499d2dc1c4b84a988e271f00c3c4160e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/c41d71974c7e43778f9226e5ea2a5e7d] 2024-11-15T00:51:02,907 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:51:02,907 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/C is initiating minor compaction (all files) 2024-11-15T00:51:02,907 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/C in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,907 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4beb458d261c4a6198877f0b03738ae8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/86683b2c27834ba59cf460522f629adc, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f6dd7317f3b54d4c846e86f4219fc6ab, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a318135540bf4b00a9358d0c2280b8c5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/dd14a784626d4497943f470fbeb42dce] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=58.7 K 2024-11-15T00:51:02,908 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48eaa8142e1242489e25f24eca5ee9d8, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731631856278 2024-11-15T00:51:02,908 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4beb458d261c4a6198877f0b03738ae8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731631856278 2024-11-15T00:51:02,908 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 70c1c743ea5448d2a5e47d6e5e7247ac, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731631856331 2024-11-15T00:51:02,908 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 86683b2c27834ba59cf460522f629adc, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731631856331 2024-11-15T00:51:02,908 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 29ce6c68d5ae4e00a31e78fc93d0327a, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731631857484 2024-11-15T00:51:02,909 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f6dd7317f3b54d4c846e86f4219fc6ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731631857484 2024-11-15T00:51:02,909 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 499d2dc1c4b84a988e271f00c3c4160e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731631858667 2024-11-15T00:51:02,909 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a318135540bf4b00a9358d0c2280b8c5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731631858667 2024-11-15T00:51:02,909 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting c41d71974c7e43778f9226e5ea2a5e7d, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631860847 2024-11-15T00:51:02,910 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting dd14a784626d4497943f470fbeb42dce, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631860847 2024-11-15T00:51:02,935 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#C#compaction#469 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:02,936 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/4adda3196d804f89953e37cf8d9ea678 is 50, key is test_row_0/C:col10/1731631860852/Put/seqid=0 2024-11-15T00:51:02,941 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:02,943 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-15T00:51:02,944 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:02,944 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T00:51:02,959 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115987d8772aa2642cb833b1b82760823ea_4c1752556c034b2ca5cdf7bf0db8cd94 store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:02,962 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115987d8772aa2642cb833b1b82760823ea_4c1752556c034b2ca5cdf7bf0db8cd94, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:02,963 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115987d8772aa2642cb833b1b82760823ea_4c1752556c034b2ca5cdf7bf0db8cd94 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:03,000 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742382_1558 (size=12325) 2024-11-15T00:51:03,006 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f6a75ff6c27049f3a912e03e22a96220_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631861534/Put/seqid=0 2024-11-15T00:51:03,021 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742383_1559 (size=4469) 2024-11-15T00:51:03,051 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742384_1560 (size=12304) 2024-11-15T00:51:03,448 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/4adda3196d804f89953e37cf8d9ea678 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4adda3196d804f89953e37cf8d9ea678 2024-11-15T00:51:03,453 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#A#compaction#470 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:03,453 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/f8a10eb4311144be848b621c0eb4be26 is 175, key is test_row_0/A:col10/1731631860852/Put/seqid=0 2024-11-15T00:51:03,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:03,479 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742385_1561 (size=31279) 2024-11-15T00:51:03,483 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f6a75ff6c27049f3a912e03e22a96220_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f6a75ff6c27049f3a912e03e22a96220_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:03,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/d46d35a6472c4624aba30ae573469904, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:03,488 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/d46d35a6472c4624aba30ae573469904 is 175, key is test_row_0/A:col10/1731631861534/Put/seqid=0 2024-11-15T00:51:03,499 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/f8a10eb4311144be848b621c0eb4be26 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f8a10eb4311144be848b621c0eb4be26 2024-11-15T00:51:03,500 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/C of 4c1752556c034b2ca5cdf7bf0db8cd94 into 4adda3196d804f89953e37cf8d9ea678(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:03,500 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:03,500 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/C, priority=11, startTime=1731631862904; duration=0sec 2024-11-15T00:51:03,501 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:51:03,501 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:C 2024-11-15T00:51:03,501 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:51:03,507 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742386_1562 (size=31105) 2024-11-15T00:51:03,508 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60158 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:51:03,508 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/B is initiating minor compaction (all files) 2024-11-15T00:51:03,508 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/B in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:03,508 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ce66d56c72674cb791b753d302b58bca, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/56b11721848b46ba9e3c0a67cd03c523, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/5cb131d640f844ba913cda004d30744d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/627b05101ece44a6be5e5f582e1a008e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/53d39b4fcc634d37b146c48b780ee9d0] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=58.7 K 2024-11-15T00:51:03,512 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ce66d56c72674cb791b753d302b58bca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1731631856278 2024-11-15T00:51:03,513 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 56b11721848b46ba9e3c0a67cd03c523, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=81, earliestPutTs=1731631856331 2024-11-15T00:51:03,513 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 5cb131d640f844ba913cda004d30744d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=92, earliestPutTs=1731631857484 2024-11-15T00:51:03,513 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 627b05101ece44a6be5e5f582e1a008e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1731631858667 2024-11-15T00:51:03,514 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 53d39b4fcc634d37b146c48b780ee9d0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631860847 2024-11-15T00:51:03,514 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/A of 4c1752556c034b2ca5cdf7bf0db8cd94 into f8a10eb4311144be848b621c0eb4be26(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:03,514 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:03,514 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/A, priority=11, startTime=1731631862903; duration=0sec 2024-11-15T00:51:03,515 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:03,515 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:A 2024-11-15T00:51:03,531 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#B#compaction#472 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:03,532 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/8531e99fa6044f4aac633f0901ea005a is 50, key is test_row_0/B:col10/1731631860852/Put/seqid=0 2024-11-15T00:51:03,576 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742387_1563 (size=12325) 2024-11-15T00:51:03,613 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/8531e99fa6044f4aac633f0901ea005a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8531e99fa6044f4aac633f0901ea005a 2024-11-15T00:51:03,660 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/B of 4c1752556c034b2ca5cdf7bf0db8cd94 into 8531e99fa6044f4aac633f0901ea005a(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:03,661 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:03,661 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/B, priority=11, startTime=1731631862904; duration=0sec 2024-11-15T00:51:03,661 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:03,661 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:B 2024-11-15T00:51:03,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T00:51:03,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:03,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:03,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631923718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:03,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631923718, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:03,723 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631923719, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,723 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:03,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631923827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:03,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,829 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:03,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631923827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:03,829 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:03,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631923827, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:03,830 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:03,909 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=153, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/d46d35a6472c4624aba30ae573469904 2024-11-15T00:51:03,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/490da2fc561c4d94b175c7f55039869c is 50, key is test_row_0/B:col10/1731631861534/Put/seqid=0 2024-11-15T00:51:03,944 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742388_1564 (size=12151) 2024-11-15T00:51:04,038 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:04,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631924036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:04,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631924036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:04,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:04,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631924036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,039 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:04,345 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/490da2fc561c4d94b175c7f55039869c 2024-11-15T00:51:04,348 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:04,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631924347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:04,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:04,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631924348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,350 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:04,351 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:04,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631924348, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:04,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:04,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/00d46966bb614bdaa3ebf1838075b4fd is 50, key is test_row_0/C:col10/1731631861534/Put/seqid=0 2024-11-15T00:51:04,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742389_1565 (size=12151) 2024-11-15T00:51:04,764 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=153 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/00d46966bb614bdaa3ebf1838075b4fd 2024-11-15T00:51:04,773 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/d46d35a6472c4624aba30ae573469904 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/d46d35a6472c4624aba30ae573469904 2024-11-15T00:51:04,786 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/d46d35a6472c4624aba30ae573469904, entries=150, sequenceid=153, filesize=30.4 K 2024-11-15T00:51:04,787 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/490da2fc561c4d94b175c7f55039869c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/490da2fc561c4d94b175c7f55039869c 2024-11-15T00:51:04,794 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/490da2fc561c4d94b175c7f55039869c, entries=150, sequenceid=153, filesize=11.9 K 2024-11-15T00:51:04,796 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/00d46966bb614bdaa3ebf1838075b4fd as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/00d46966bb614bdaa3ebf1838075b4fd 2024-11-15T00:51:04,799 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/00d46966bb614bdaa3ebf1838075b4fd, entries=150, sequenceid=153, filesize=11.9 K 2024-11-15T00:51:04,800 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1856ms, sequenceid=153, compaction requested=false 2024-11-15T00:51:04,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:04,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:04,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-15T00:51:04,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-15T00:51:04,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-15T00:51:04,804 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.2510 sec 2024-11-15T00:51:04,806 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 3.2570 sec 2024-11-15T00:51:04,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:04,858 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:51:04,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T00:51:04,871 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111503ad76cf7d0e48fdb641d68f933b2968_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631864857/Put/seqid=0 2024-11-15T00:51:04,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742390_1566 (size=17284) 2024-11-15T00:51:04,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:04,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631924905, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,910 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:04,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:04,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631924908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:04,912 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:04,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631924908, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:04,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,913 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:04,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,913 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:04,914 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631925017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631925017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631925017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,020 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,229 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631925227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631925227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,230 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,230 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631925227, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,278 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:05,281 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111503ad76cf7d0e48fdb641d68f933b2968_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111503ad76cf7d0e48fdb641d68f933b2968_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:05,282 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/e52de19f37b2472394449e24136d16c3, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:05,282 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/e52de19f37b2472394449e24136d16c3 is 175, key is test_row_0/A:col10/1731631864857/Put/seqid=0 2024-11-15T00:51:05,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742391_1567 (size=48389) 2024-11-15T00:51:05,290 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=170, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/e52de19f37b2472394449e24136d16c3 2024-11-15T00:51:05,300 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/26fd1e4ae51a469faf81e4f11cded4d3 is 50, key is test_row_0/B:col10/1731631864857/Put/seqid=0 2024-11-15T00:51:05,309 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742392_1568 (size=12151) 2024-11-15T00:51:05,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/26fd1e4ae51a469faf81e4f11cded4d3 2024-11-15T00:51:05,315 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/daea7fcc67774a7b9c221d1cbca8afdf is 50, key is test_row_0/C:col10/1731631864857/Put/seqid=0 2024-11-15T00:51:05,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742393_1569 (size=12151) 2024-11-15T00:51:05,326 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=170 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/daea7fcc67774a7b9c221d1cbca8afdf 2024-11-15T00:51:05,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/e52de19f37b2472394449e24136d16c3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/e52de19f37b2472394449e24136d16c3 2024-11-15T00:51:05,342 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/e52de19f37b2472394449e24136d16c3, entries=250, sequenceid=170, filesize=47.3 K 2024-11-15T00:51:05,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/26fd1e4ae51a469faf81e4f11cded4d3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/26fd1e4ae51a469faf81e4f11cded4d3 2024-11-15T00:51:05,350 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/26fd1e4ae51a469faf81e4f11cded4d3, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T00:51:05,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/daea7fcc67774a7b9c221d1cbca8afdf as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/daea7fcc67774a7b9c221d1cbca8afdf 2024-11-15T00:51:05,366 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/daea7fcc67774a7b9c221d1cbca8afdf, entries=150, sequenceid=170, filesize=11.9 K 2024-11-15T00:51:05,367 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 509ms, sequenceid=170, compaction requested=true 2024-11-15T00:51:05,367 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:05,367 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:51:05,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:51:05,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:05,368 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:51:05,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:51:05,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:05,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:51:05,368 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:51:05,377 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36627 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:51:05,377 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/B is initiating minor compaction (all files) 2024-11-15T00:51:05,377 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/B in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:05,377 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8531e99fa6044f4aac633f0901ea005a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/490da2fc561c4d94b175c7f55039869c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/26fd1e4ae51a469faf81e4f11cded4d3] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=35.8 K 2024-11-15T00:51:05,378 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110773 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:51:05,378 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/A is initiating minor compaction (all files) 2024-11-15T00:51:05,378 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/A in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:05,378 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f8a10eb4311144be848b621c0eb4be26, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/d46d35a6472c4624aba30ae573469904, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/e52de19f37b2472394449e24136d16c3] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=108.2 K 2024-11-15T00:51:05,378 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8531e99fa6044f4aac633f0901ea005a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631860847 2024-11-15T00:51:05,378 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:05,378 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f8a10eb4311144be848b621c0eb4be26, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/d46d35a6472c4624aba30ae573469904, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/e52de19f37b2472394449e24136d16c3] 2024-11-15T00:51:05,378 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 490da2fc561c4d94b175c7f55039869c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731631861529 2024-11-15T00:51:05,378 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting f8a10eb4311144be848b621c0eb4be26, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631860847 2024-11-15T00:51:05,378 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 26fd1e4ae51a469faf81e4f11cded4d3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631863716 2024-11-15T00:51:05,379 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting d46d35a6472c4624aba30ae573469904, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731631861529 2024-11-15T00:51:05,379 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting e52de19f37b2472394449e24136d16c3, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631863715 2024-11-15T00:51:05,391 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#B#compaction#478 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:05,391 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/ddcf1208e1014a39a09a307a927088d4 is 50, key is test_row_0/B:col10/1731631864857/Put/seqid=0 2024-11-15T00:51:05,397 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:05,405 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411157e2f5e9daa894e7c940882426818a630_4c1752556c034b2ca5cdf7bf0db8cd94 store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:05,407 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411157e2f5e9daa894e7c940882426818a630_4c1752556c034b2ca5cdf7bf0db8cd94, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:05,408 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411157e2f5e9daa894e7c940882426818a630_4c1752556c034b2ca5cdf7bf0db8cd94 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:05,409 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742394_1570 (size=12527) 2024-11-15T00:51:05,412 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742395_1571 (size=4469) 2024-11-15T00:51:05,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:05,549 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:51:05,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T00:51:05,566 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ca915061b9964701ab260f97bc2e624e_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631865541/Put/seqid=0 2024-11-15T00:51:05,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631925584, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,588 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,592 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631925585, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,593 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,593 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742396_1572 (size=12304) 2024-11-15T00:51:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631925586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,594 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,594 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:05,598 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115ca915061b9964701ab260f97bc2e624e_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ca915061b9964701ab260f97bc2e624e_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:05,599 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/0bdd92d9797e44b8bdc8d169301532d8, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:05,600 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/0bdd92d9797e44b8bdc8d169301532d8 is 175, key is test_row_0/A:col10/1731631865541/Put/seqid=0 2024-11-15T00:51:05,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742397_1573 (size=31105) 2024-11-15T00:51:05,637 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/0bdd92d9797e44b8bdc8d169301532d8 2024-11-15T00:51:05,649 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/fdaca34fe5784f0487bdf55e28d18cfa is 50, key is test_row_0/B:col10/1731631865541/Put/seqid=0 2024-11-15T00:51:05,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742398_1574 (size=12151) 2024-11-15T00:51:05,692 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/fdaca34fe5784f0487bdf55e28d18cfa 2024-11-15T00:51:05,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631925698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631925698, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,700 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,701 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:05,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631925700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:05,704 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:05,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/b7df6ad5b76d4cb29c6975e648b23c84 is 50, key is test_row_0/C:col10/1731631865541/Put/seqid=0 2024-11-15T00:51:05,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-15T00:51:05,717 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:51:05,717 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:51:05,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-15T00:51:05,726 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:51:05,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T00:51:05,726 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:51:05,727 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:51:05,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742399_1575 (size=12151) 2024-11-15T00:51:05,743 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/b7df6ad5b76d4cb29c6975e648b23c84 2024-11-15T00:51:05,752 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/0bdd92d9797e44b8bdc8d169301532d8 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/0bdd92d9797e44b8bdc8d169301532d8 2024-11-15T00:51:05,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/0bdd92d9797e44b8bdc8d169301532d8, entries=150, sequenceid=195, filesize=30.4 K 2024-11-15T00:51:05,772 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/fdaca34fe5784f0487bdf55e28d18cfa as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/fdaca34fe5784f0487bdf55e28d18cfa 2024-11-15T00:51:05,809 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/fdaca34fe5784f0487bdf55e28d18cfa, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T00:51:05,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/b7df6ad5b76d4cb29c6975e648b23c84 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b7df6ad5b76d4cb29c6975e648b23c84 2024-11-15T00:51:05,818 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#A#compaction#479 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:05,819 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/8bcece740f6d46b899a32cf070421c76 is 175, key is test_row_0/A:col10/1731631864857/Put/seqid=0 2024-11-15T00:51:05,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T00:51:05,839 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b7df6ad5b76d4cb29c6975e648b23c84, entries=150, sequenceid=195, filesize=11.9 K 2024-11-15T00:51:05,840 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 290ms, sequenceid=195, compaction requested=true 2024-11-15T00:51:05,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:05,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:A, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:51:05,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:51:05,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:B, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:51:05,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=3), splitQueue=0 2024-11-15T00:51:05,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:51:05,840 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T00:51:05,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742400_1576 (size=31481) 2024-11-15T00:51:05,878 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/ddcf1208e1014a39a09a307a927088d4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ddcf1208e1014a39a09a307a927088d4 2024-11-15T00:51:05,879 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-15T00:51:05,880 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:05,880 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-15T00:51:05,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158daa04502444456895bea03064d497bb_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631865582/Put/seqid=0 2024-11-15T00:51:05,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:05,914 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:05,917 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/B of 4c1752556c034b2ca5cdf7bf0db8cd94 into ddcf1208e1014a39a09a307a927088d4(size=12.2 K), total size for store is 24.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:05,918 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:05,918 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/B, priority=13, startTime=1731631865368; duration=0sec 2024-11-15T00:51:05,918 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=4), splitQueue=0 2024-11-15T00:51:05,918 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:B 2024-11-15T00:51:05,918 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 3 compacting, 1 eligible, 16 blocking 2024-11-15T00:51:05,919 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:51:05,919 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:51:05,920 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. because compaction request was cancelled 2024-11-15T00:51:05,920 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:A 2024-11-15T00:51:05,920 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:51:05,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742401_1577 (size=12304) 2024-11-15T00:51:05,929 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48778 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:51:05,929 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/C is initiating minor compaction (all files) 2024-11-15T00:51:05,929 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/C in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:05,929 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4adda3196d804f89953e37cf8d9ea678, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/00d46966bb614bdaa3ebf1838075b4fd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/daea7fcc67774a7b9c221d1cbca8afdf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b7df6ad5b76d4cb29c6975e648b23c84] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=47.6 K 2024-11-15T00:51:05,936 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 4adda3196d804f89953e37cf8d9ea678, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=129, earliestPutTs=1731631860847 2024-11-15T00:51:05,937 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 00d46966bb614bdaa3ebf1838075b4fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=153, earliestPutTs=1731631861529 2024-11-15T00:51:05,937 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting daea7fcc67774a7b9c221d1cbca8afdf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631863716 2024-11-15T00:51:05,938 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b7df6ad5b76d4cb29c6975e648b23c84, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731631864905 2024-11-15T00:51:05,970 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#C#compaction#484 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:05,971 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/b3764ba45bad4430bbda5db81cc02dce is 50, key is test_row_0/C:col10/1731631865541/Put/seqid=0 2024-11-15T00:51:05,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742402_1578 (size=12561) 2024-11-15T00:51:05,987 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/b3764ba45bad4430bbda5db81cc02dce as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b3764ba45bad4430bbda5db81cc02dce 2024-11-15T00:51:06,003 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/C of 4c1752556c034b2ca5cdf7bf0db8cd94 into b3764ba45bad4430bbda5db81cc02dce(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:06,003 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:06,003 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/C, priority=12, startTime=1731631865840; duration=0sec 2024-11-15T00:51:06,003 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:51:06,003 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:C 2024-11-15T00:51:06,003 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:B 2024-11-15T00:51:06,003 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:C 2024-11-15T00:51:06,004 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-15T00:51:06,005 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:51:06,005 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:51:06,005 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. because compaction request was cancelled 2024-11-15T00:51:06,005 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:B 2024-11-15T00:51:06,005 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 1 store files, 0 compacting, 1 eligible, 16 blocking 2024-11-15T00:51:06,005 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-15T00:51:06,005 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-15T00:51:06,005 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. because compaction request was cancelled 2024-11-15T00:51:06,005 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:C 2024-11-15T00:51:06,010 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631926004, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,010 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631926007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,014 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631926009, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,015 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,016 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,017 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T00:51:06,121 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631926118, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,122 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,127 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,127 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631926126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,129 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,132 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,133 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631926129, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,136 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,270 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/8bcece740f6d46b899a32cf070421c76 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/8bcece740f6d46b899a32cf070421c76 2024-11-15T00:51:06,279 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/A of 4c1752556c034b2ca5cdf7bf0db8cd94 into 8bcece740f6d46b899a32cf070421c76(size=30.7 K), total size for store is 61.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:06,279 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:06,279 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/A, priority=13, startTime=1731631865367; duration=0sec 2024-11-15T00:51:06,279 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:06,279 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:A 2024-11-15T00:51:06,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:06,329 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158daa04502444456895bea03064d497bb_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158daa04502444456895bea03064d497bb_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:06,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/106b42f8190c4f7084d6b0813af66002, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:06,330 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/106b42f8190c4f7084d6b0813af66002 is 175, key is test_row_0/A:col10/1731631865582/Put/seqid=0 2024-11-15T00:51:06,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631926327, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,335 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742403_1579 (size=31105) 2024-11-15T00:51:06,340 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631926336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,349 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631926346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,349 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T00:51:06,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631926636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631926646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,649 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,658 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631926657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,659 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,736 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=206, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/106b42f8190c4f7084d6b0813af66002 2024-11-15T00:51:06,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/522fe394e52c49cda53f0b0de1886c75 is 50, key is test_row_0/B:col10/1731631865582/Put/seqid=0 2024-11-15T00:51:06,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742404_1580 (size=12151) 2024-11-15T00:51:06,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T00:51:06,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33616 deadline: 1731631926917, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,921 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_1' in 4c1752556c034b2ca5cdf7bf0db8cd94 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8245 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:51:06,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:06,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:06,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33602 deadline: 1731631926929, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:06,934 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to fb2f6cb7846a,42925,1731631698709 for 'test_row_2' in 4c1752556c034b2ca5cdf7bf0db8cd94 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8263 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-15T00:51:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:06,934 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:07,155 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:07,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631927152, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:07,156 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:07,166 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:07,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631927161, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:07,167 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:07,167 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:07,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631927167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:07,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:07,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:07,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:07,203 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/522fe394e52c49cda53f0b0de1886c75 2024-11-15T00:51:07,228 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/2174c2580e32404ba9f82a22234e994e is 50, key is test_row_0/C:col10/1731631865582/Put/seqid=0 2024-11-15T00:51:07,256 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742405_1581 (size=12151) 2024-11-15T00:51:07,654 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=206 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/2174c2580e32404ba9f82a22234e994e 2024-11-15T00:51:07,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/106b42f8190c4f7084d6b0813af66002 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/106b42f8190c4f7084d6b0813af66002 2024-11-15T00:51:07,663 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/106b42f8190c4f7084d6b0813af66002, entries=150, sequenceid=206, filesize=30.4 K 2024-11-15T00:51:07,664 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/522fe394e52c49cda53f0b0de1886c75 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/522fe394e52c49cda53f0b0de1886c75 2024-11-15T00:51:07,668 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/522fe394e52c49cda53f0b0de1886c75, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T00:51:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/2174c2580e32404ba9f82a22234e994e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2174c2580e32404ba9f82a22234e994e 2024-11-15T00:51:07,676 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2174c2580e32404ba9f82a22234e994e, entries=150, sequenceid=206, filesize=11.9 K 2024-11-15T00:51:07,678 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1797ms, sequenceid=206, compaction requested=true 2024-11-15T00:51:07,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:07,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:07,678 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-15T00:51:07,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-15T00:51:07,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-15T00:51:07,688 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9580 sec 2024-11-15T00:51:07,693 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 1.9740 sec 2024-11-15T00:51:07,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-15T00:51:07,877 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:51:07,878 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:51:07,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-15T00:51:07,880 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:51:07,883 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:51:07,883 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:51:07,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T00:51:07,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T00:51:08,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-15T00:51:08,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:08,037 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-15T00:51:08,086 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111506cb38e086614d47b2d94d663b48042b_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631866001/Put/seqid=0 2024-11-15T00:51:08,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742406_1582 (size=12304) 2024-11-15T00:51:08,118 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:08,124 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111506cb38e086614d47b2d94d663b48042b_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111506cb38e086614d47b2d94d663b48042b_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:08,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/893dae7e68414410b266362324f769de, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:08,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/893dae7e68414410b266362324f769de is 175, key is test_row_0/A:col10/1731631866001/Put/seqid=0 2024-11-15T00:51:08,152 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742407_1583 (size=31105) 2024-11-15T00:51:08,153 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=234, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/893dae7e68414410b266362324f769de 2024-11-15T00:51:08,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:08,188 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:08,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T00:51:08,214 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/a1b9508ff81346048dd7cbba570b14f1 is 50, key is test_row_0/B:col10/1731631866001/Put/seqid=0 2024-11-15T00:51:08,240 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631928236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631928240, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,249 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,253 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631928245, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,254 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742408_1584 (size=12151) 2024-11-15T00:51:08,351 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631928347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,351 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,352 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631928357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631928357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,365 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T00:51:08,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631928559, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,562 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,578 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631928577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,579 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631928577, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,579 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,669 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/a1b9508ff81346048dd7cbba570b14f1 2024-11-15T00:51:08,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/a3a3cd0bf23c46afbb958571fd19b491 is 50, key is test_row_0/C:col10/1731631866001/Put/seqid=0 2024-11-15T00:51:08,770 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742409_1585 (size=12151) 2024-11-15T00:51:08,784 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/a3a3cd0bf23c46afbb958571fd19b491 2024-11-15T00:51:08,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/893dae7e68414410b266362324f769de as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/893dae7e68414410b266362324f769de 2024-11-15T00:51:08,801 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/893dae7e68414410b266362324f769de, entries=150, sequenceid=234, filesize=30.4 K 2024-11-15T00:51:08,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/a1b9508ff81346048dd7cbba570b14f1 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/a1b9508ff81346048dd7cbba570b14f1 2024-11-15T00:51:08,810 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/a1b9508ff81346048dd7cbba570b14f1, entries=150, sequenceid=234, filesize=11.9 K 2024-11-15T00:51:08,817 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/a3a3cd0bf23c46afbb958571fd19b491 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a3a3cd0bf23c46afbb958571fd19b491 2024-11-15T00:51:08,822 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a3a3cd0bf23c46afbb958571fd19b491, entries=150, sequenceid=234, filesize=11.9 K 2024-11-15T00:51:08,828 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 791ms, sequenceid=234, compaction requested=true 2024-11-15T00:51:08,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:08,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:08,829 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-15T00:51:08,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-15T00:51:08,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-15T00:51:08,835 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 950 msec 2024-11-15T00:51:08,838 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 958 msec 2024-11-15T00:51:08,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:08,882 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:51:08,882 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-15T00:51:08,915 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c15cd4f0e97945d087c6455b79228a41_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631868212/Put/seqid=0 2024-11-15T00:51:08,935 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742410_1586 (size=12304) 2024-11-15T00:51:08,972 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631928968, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,973 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631928972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,979 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:08,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631928973, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:08,980 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:08,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:09,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-15T00:51:09,028 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:51:09,029 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:51:09,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees 2024-11-15T00:51:09,031 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:51:09,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T00:51:09,032 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=164, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:51:09,032 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:51:09,082 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:09,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631929080, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:09,082 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,083 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:09,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:09,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631929087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:09,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:09,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631929087, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:09,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T00:51:09,205 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T00:51:09,205 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:09,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:09,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:09,206 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,206 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,301 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:09,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631929292, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,302 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:09,314 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:09,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:09,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631929309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:09,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,315 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,316 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:09,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631929309, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:09,336 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:09,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T00:51:09,357 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115c15cd4f0e97945d087c6455b79228a41_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c15cd4f0e97945d087c6455b79228a41_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:09,361 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/a8c5870408074a558bd7fd2448e07d80, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:09,362 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/a8c5870408074a558bd7fd2448e07d80 is 175, key is test_row_0/A:col10/1731631868212/Put/seqid=0 2024-11-15T00:51:09,368 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T00:51:09,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:09,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:09,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:09,372 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,372 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,393 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742411_1587 (size=31105) 2024-11-15T00:51:09,397 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=246, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/a8c5870408074a558bd7fd2448e07d80 2024-11-15T00:51:09,446 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/13fc7a90657a463e8b8e74f8629116cb is 50, key is test_row_0/B:col10/1731631868212/Put/seqid=0 2024-11-15T00:51:09,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742412_1588 (size=12151) 2024-11-15T00:51:09,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T00:51:09,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:09,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:09,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:09,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631929657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:09,663 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:09,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631929658, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:09,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,664 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,665 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:09,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,665 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:09,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T00:51:09,674 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:09,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631929664, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:09,674 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:09,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T00:51:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:09,704 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:09,705 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,705 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,868 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T00:51:09,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:09,868 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:09,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:09,908 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/13fc7a90657a463e8b8e74f8629116cb 2024-11-15T00:51:09,950 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/1488332f5a5b4efaac4886e640dba11b is 50, key is test_row_0/C:col10/1731631868212/Put/seqid=0 2024-11-15T00:51:10,012 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742413_1589 (size=12151) 2024-11-15T00:51:10,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=246 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/1488332f5a5b4efaac4886e640dba11b 2024-11-15T00:51:10,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T00:51:10,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:10,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:10,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:10,052 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:10,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:10,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:10,098 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/a8c5870408074a558bd7fd2448e07d80 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a8c5870408074a558bd7fd2448e07d80 2024-11-15T00:51:10,150 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a8c5870408074a558bd7fd2448e07d80, entries=150, sequenceid=246, filesize=30.4 K 2024-11-15T00:51:10,160 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/13fc7a90657a463e8b8e74f8629116cb as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/13fc7a90657a463e8b8e74f8629116cb 2024-11-15T00:51:10,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:10,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631930167, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:10,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:10,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:10,172 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:10,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T00:51:10,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:10,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631930168, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:10,179 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:10,194 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:10,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631930188, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:10,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:10,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:10,194 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:10,201 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/13fc7a90657a463e8b8e74f8629116cb, entries=150, sequenceid=246, filesize=11.9 K 2024-11-15T00:51:10,216 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/1488332f5a5b4efaac4886e640dba11b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/1488332f5a5b4efaac4886e640dba11b 2024-11-15T00:51:10,219 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T00:51:10,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:10,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:10,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:10,219 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] handler.RSProcedureHandler(58): pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:10,219 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=165 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:10,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=165 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:10,240 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/1488332f5a5b4efaac4886e640dba11b, entries=150, sequenceid=246, filesize=11.9 K 2024-11-15T00:51:10,241 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1359ms, sequenceid=246, compaction requested=true 2024-11-15T00:51:10,241 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:10,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:51:10,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:10,241 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:51:10,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:51:10,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:51:10,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:51:10,241 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:51:10,242 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-15T00:51:10,261 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 61131 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:51:10,261 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/B is initiating minor compaction (all files) 2024-11-15T00:51:10,261 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/B in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:10,261 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ddcf1208e1014a39a09a307a927088d4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/fdaca34fe5784f0487bdf55e28d18cfa, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/522fe394e52c49cda53f0b0de1886c75, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/a1b9508ff81346048dd7cbba570b14f1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/13fc7a90657a463e8b8e74f8629116cb] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=59.7 K 2024-11-15T00:51:10,264 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 155901 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-15T00:51:10,264 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/A is initiating minor compaction (all files) 2024-11-15T00:51:10,264 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/A in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:10,264 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/8bcece740f6d46b899a32cf070421c76, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/0bdd92d9797e44b8bdc8d169301532d8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/106b42f8190c4f7084d6b0813af66002, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/893dae7e68414410b266362324f769de, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a8c5870408074a558bd7fd2448e07d80] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=152.2 K 2024-11-15T00:51:10,265 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:10,265 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/8bcece740f6d46b899a32cf070421c76, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/0bdd92d9797e44b8bdc8d169301532d8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/106b42f8190c4f7084d6b0813af66002, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/893dae7e68414410b266362324f769de, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a8c5870408074a558bd7fd2448e07d80] 2024-11-15T00:51:10,266 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ddcf1208e1014a39a09a307a927088d4, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631863716 2024-11-15T00:51:10,272 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8bcece740f6d46b899a32cf070421c76, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=170, earliestPutTs=1731631863716 2024-11-15T00:51:10,273 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting fdaca34fe5784f0487bdf55e28d18cfa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731631864905 2024-11-15T00:51:10,276 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bdd92d9797e44b8bdc8d169301532d8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731631864905 2024-11-15T00:51:10,276 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 522fe394e52c49cda53f0b0de1886c75, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631865580 2024-11-15T00:51:10,277 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a1b9508ff81346048dd7cbba570b14f1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731631866001 2024-11-15T00:51:10,280 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 106b42f8190c4f7084d6b0813af66002, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631865580 2024-11-15T00:51:10,282 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 13fc7a90657a463e8b8e74f8629116cb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731631868197 2024-11-15T00:51:10,284 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 893dae7e68414410b266362324f769de, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731631866001 2024-11-15T00:51:10,288 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8c5870408074a558bd7fd2448e07d80, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731631868197 2024-11-15T00:51:10,327 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#B#compaction#493 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:10,327 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/8da5cb5f7151449c800d94691b38d18f is 50, key is test_row_0/B:col10/1731631868212/Put/seqid=0 2024-11-15T00:51:10,341 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:10,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742414_1590 (size=12697) 2024-11-15T00:51:10,369 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111507c6db8dc9044479893d1d881fcc4d17_4c1752556c034b2ca5cdf7bf0db8cd94 store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:10,372 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111507c6db8dc9044479893d1d881fcc4d17_4c1752556c034b2ca5cdf7bf0db8cd94, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:10,372 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111507c6db8dc9044479893d1d881fcc4d17_4c1752556c034b2ca5cdf7bf0db8cd94 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:10,376 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=165 2024-11-15T00:51:10,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:10,376 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T00:51:10,379 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742415_1591 (size=4469) 2024-11-15T00:51:10,390 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bf8552c51d5740e6a27bd614b15652a4_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631868970/Put/seqid=0 2024-11-15T00:51:10,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742416_1592 (size=12454) 2024-11-15T00:51:10,445 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:10,461 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115bf8552c51d5740e6a27bd614b15652a4_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf8552c51d5740e6a27bd614b15652a4_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:10,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/896927db12d5456589d9f515cb3adbcb, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:10,463 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/896927db12d5456589d9f515cb3adbcb is 175, key is test_row_0/A:col10/1731631868970/Put/seqid=0 2024-11-15T00:51:10,505 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742417_1593 (size=31255) 2024-11-15T00:51:10,507 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=270, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/896927db12d5456589d9f515cb3adbcb 2024-11-15T00:51:10,534 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/33a571d501b440338836188f65b282a3 is 50, key is test_row_0/B:col10/1731631868970/Put/seqid=0 2024-11-15T00:51:10,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742418_1594 (size=12301) 2024-11-15T00:51:10,785 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#A#compaction#494 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:10,785 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/63e12e73e8484a1b93ee88ef57ee97bb is 175, key is test_row_0/A:col10/1731631868212/Put/seqid=0 2024-11-15T00:51:10,788 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/8da5cb5f7151449c800d94691b38d18f as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8da5cb5f7151449c800d94691b38d18f 2024-11-15T00:51:10,808 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/B of 4c1752556c034b2ca5cdf7bf0db8cd94 into 8da5cb5f7151449c800d94691b38d18f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:10,809 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:10,809 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/B, priority=11, startTime=1731631870241; duration=0sec 2024-11-15T00:51:10,809 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:51:10,809 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:B 2024-11-15T00:51:10,809 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-15T00:51:10,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742419_1595 (size=31651) 2024-11-15T00:51:10,811 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49014 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-15T00:51:10,811 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/C is initiating minor compaction (all files) 2024-11-15T00:51:10,811 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/C in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:10,811 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b3764ba45bad4430bbda5db81cc02dce, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2174c2580e32404ba9f82a22234e994e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a3a3cd0bf23c46afbb958571fd19b491, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/1488332f5a5b4efaac4886e640dba11b] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=47.9 K 2024-11-15T00:51:10,811 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting b3764ba45bad4430bbda5db81cc02dce, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1731631864905 2024-11-15T00:51:10,812 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 2174c2580e32404ba9f82a22234e994e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=206, earliestPutTs=1731631865580 2024-11-15T00:51:10,812 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting a3a3cd0bf23c46afbb958571fd19b491, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1731631866001 2024-11-15T00:51:10,818 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 1488332f5a5b4efaac4886e640dba11b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731631868197 2024-11-15T00:51:10,859 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#C#compaction#497 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:10,860 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/16b6a0d510bf4f048afafed079dac53d is 50, key is test_row_0/C:col10/1731631868212/Put/seqid=0 2024-11-15T00:51:10,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742420_1596 (size=12697) 2024-11-15T00:51:10,988 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/33a571d501b440338836188f65b282a3 2024-11-15T00:51:11,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/f825d5503d554297af2fbe720b84df6a is 50, key is test_row_0/C:col10/1731631868970/Put/seqid=0 2024-11-15T00:51:11,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742421_1597 (size=12301) 2024-11-15T00:51:11,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T00:51:11,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:11,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:11,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631931216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,223 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631931216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,223 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,224 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,230 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631931223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,231 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,235 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/63e12e73e8484a1b93ee88ef57ee97bb as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/63e12e73e8484a1b93ee88ef57ee97bb 2024-11-15T00:51:11,254 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/A of 4c1752556c034b2ca5cdf7bf0db8cd94 into 63e12e73e8484a1b93ee88ef57ee97bb(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:11,254 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:11,254 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/A, priority=11, startTime=1731631870241; duration=0sec 2024-11-15T00:51:11,254 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:11,254 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:A 2024-11-15T00:51:11,338 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631931332, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631931335, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,340 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,341 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631931338, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,342 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,343 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,345 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/16b6a0d510bf4f048afafed079dac53d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/16b6a0d510bf4f048afafed079dac53d 2024-11-15T00:51:11,375 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/C of 4c1752556c034b2ca5cdf7bf0db8cd94 into 16b6a0d510bf4f048afafed079dac53d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:11,375 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:11,375 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/C, priority=12, startTime=1731631870241; duration=0sec 2024-11-15T00:51:11,375 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:11,376 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:C 2024-11-15T00:51:11,490 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=270 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/f825d5503d554297af2fbe720b84df6a 2024-11-15T00:51:11,518 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/896927db12d5456589d9f515cb3adbcb as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/896927db12d5456589d9f515cb3adbcb 2024-11-15T00:51:11,534 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/896927db12d5456589d9f515cb3adbcb, entries=150, sequenceid=270, filesize=30.5 K 2024-11-15T00:51:11,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/33a571d501b440338836188f65b282a3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/33a571d501b440338836188f65b282a3 2024-11-15T00:51:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,546 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/33a571d501b440338836188f65b282a3, entries=150, sequenceid=270, filesize=12.0 K 2024-11-15T00:51:11,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/f825d5503d554297af2fbe720b84df6a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f825d5503d554297af2fbe720b84df6a 2024-11-15T00:51:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,553 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f825d5503d554297af2fbe720b84df6a, entries=150, sequenceid=270, filesize=12.0 K 2024-11-15T00:51:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,554 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1178ms, sequenceid=270, compaction requested=false 2024-11-15T00:51:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:11,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=165}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=165 2024-11-15T00:51:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=165 2024-11-15T00:51:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,557 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:51:11,557 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T00:51:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-15T00:51:11,558 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5230 sec 2024-11-15T00:51:11,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,561 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=164, table=TestAcidGuarantees in 2.5300 sec 2024-11-15T00:51:11,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,582 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cac801f9dad847ae8497adc111d64490_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631871213/Put/seqid=0 2024-11-15T00:51:11,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,606 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742422_1598 (size=12454) 2024-11-15T00:51:11,613 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:11,623 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115cac801f9dad847ae8497adc111d64490_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cac801f9dad847ae8497adc111d64490_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:11,630 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/7d50ceb8610042859712213af74f8b21, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:11,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/7d50ceb8610042859712213af74f8b21 is 175, key is test_row_0/A:col10/1731631871213/Put/seqid=0 2024-11-15T00:51:11,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742423_1599 (size=31255) 2024-11-15T00:51:11,652 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631931647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,653 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,658 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631931652, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631931653, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,659 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,766 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631931762, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,767 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631931772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,776 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631931772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631931978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,983 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,994 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631931994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,995 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:11,995 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:11,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631931994, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:11,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:12,037 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=286, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/7d50ceb8610042859712213af74f8b21 2024-11-15T00:51:12,061 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/66c174f59b034f01943383c81e8659a9 is 50, key is test_row_0/B:col10/1731631871213/Put/seqid=0 2024-11-15T00:51:12,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742424_1600 (size=12301) 2024-11-15T00:51:12,087 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/66c174f59b034f01943383c81e8659a9 2024-11-15T00:51:12,097 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/80a694e251c440f9b5e584dc469a247b is 50, key is test_row_0/C:col10/1731631871213/Put/seqid=0 2024-11-15T00:51:12,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742425_1601 (size=12301) 2024-11-15T00:51:12,289 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:12,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631932286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,290 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:12,307 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:12,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631932307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:12,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:12,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631932307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,308 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,309 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:12,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=286 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/80a694e251c440f9b5e584dc469a247b 2024-11-15T00:51:12,553 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/7d50ceb8610042859712213af74f8b21 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/7d50ceb8610042859712213af74f8b21 2024-11-15T00:51:12,593 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/7d50ceb8610042859712213af74f8b21, entries=150, sequenceid=286, filesize=30.5 K 2024-11-15T00:51:12,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/66c174f59b034f01943383c81e8659a9 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/66c174f59b034f01943383c81e8659a9 2024-11-15T00:51:12,657 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/66c174f59b034f01943383c81e8659a9, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T00:51:12,669 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/80a694e251c440f9b5e584dc469a247b as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/80a694e251c440f9b5e584dc469a247b 2024-11-15T00:51:12,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/80a694e251c440f9b5e584dc469a247b, entries=150, sequenceid=286, filesize=12.0 K 2024-11-15T00:51:12,697 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1139ms, sequenceid=286, compaction requested=true 2024-11-15T00:51:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:51:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:51:12,697 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:51:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:51:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:51:12,697 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-15T00:51:12,697 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:51:12,698 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:51:12,698 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/B is initiating minor compaction (all files) 2024-11-15T00:51:12,698 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/B in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:12,699 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8da5cb5f7151449c800d94691b38d18f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/33a571d501b440338836188f65b282a3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/66c174f59b034f01943383c81e8659a9] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=36.4 K 2024-11-15T00:51:12,699 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:51:12,699 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/A is initiating minor compaction (all files) 2024-11-15T00:51:12,699 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/A in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:12,699 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8da5cb5f7151449c800d94691b38d18f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731631868197 2024-11-15T00:51:12,699 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/63e12e73e8484a1b93ee88ef57ee97bb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/896927db12d5456589d9f515cb3adbcb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/7d50ceb8610042859712213af74f8b21] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=92.0 K 2024-11-15T00:51:12,699 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:12,699 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/63e12e73e8484a1b93ee88ef57ee97bb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/896927db12d5456589d9f515cb3adbcb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/7d50ceb8610042859712213af74f8b21] 2024-11-15T00:51:12,699 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 33a571d501b440338836188f65b282a3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731631868929 2024-11-15T00:51:12,700 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 63e12e73e8484a1b93ee88ef57ee97bb, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731631868197 2024-11-15T00:51:12,700 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 66c174f59b034f01943383c81e8659a9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731631871213 2024-11-15T00:51:12,700 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 896927db12d5456589d9f515cb3adbcb, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731631868929 2024-11-15T00:51:12,701 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7d50ceb8610042859712213af74f8b21, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731631871213 2024-11-15T00:51:12,740 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#B#compaction#502 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:12,740 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/8d8ba2ff8b1c4eb4a3c14e0acb2bc996 is 50, key is test_row_0/B:col10/1731631871213/Put/seqid=0 2024-11-15T00:51:12,765 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:12,767 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742426_1602 (size=12949) 2024-11-15T00:51:12,793 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241115915892d3d12b4417b0d5e36e1aeb70aa_4c1752556c034b2ca5cdf7bf0db8cd94 store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:12,795 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241115915892d3d12b4417b0d5e36e1aeb70aa_4c1752556c034b2ca5cdf7bf0db8cd94, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:12,795 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115915892d3d12b4417b0d5e36e1aeb70aa_4c1752556c034b2ca5cdf7bf0db8cd94 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:12,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:12,810 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:51:12,810 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-15T00:51:12,821 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/8d8ba2ff8b1c4eb4a3c14e0acb2bc996 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8d8ba2ff8b1c4eb4a3c14e0acb2bc996 2024-11-15T00:51:12,850 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f304bb28f6c344e988279add2c3801bd_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631871626/Put/seqid=0 2024-11-15T00:51:12,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742427_1603 (size=4469) 2024-11-15T00:51:12,862 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/B of 4c1752556c034b2ca5cdf7bf0db8cd94 into 8d8ba2ff8b1c4eb4a3c14e0acb2bc996(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:12,862 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:12,862 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/B, priority=13, startTime=1731631872697; duration=0sec 2024-11-15T00:51:12,862 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:51:12,862 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:B 2024-11-15T00:51:12,862 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:51:12,864 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:12,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631932857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:12,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631932857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:12,865 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:12,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631932857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,865 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,866 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:12,870 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37299 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:51:12,870 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/C is initiating minor compaction (all files) 2024-11-15T00:51:12,870 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/C in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:12,870 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/16b6a0d510bf4f048afafed079dac53d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f825d5503d554297af2fbe720b84df6a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/80a694e251c440f9b5e584dc469a247b] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=36.4 K 2024-11-15T00:51:12,870 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 16b6a0d510bf4f048afafed079dac53d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=246, earliestPutTs=1731631868197 2024-11-15T00:51:12,871 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting f825d5503d554297af2fbe720b84df6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=270, earliestPutTs=1731631868929 2024-11-15T00:51:12,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742428_1604 (size=14994) 2024-11-15T00:51:12,877 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 80a694e251c440f9b5e584dc469a247b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731631871213 2024-11-15T00:51:12,878 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:12,895 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115f304bb28f6c344e988279add2c3801bd_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f304bb28f6c344e988279add2c3801bd_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:12,896 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/3807f1dd562648f5814c9414cf22356d, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:12,896 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/3807f1dd562648f5814c9414cf22356d is 175, key is test_row_0/A:col10/1731631871626/Put/seqid=0 2024-11-15T00:51:12,930 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#C#compaction#505 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:12,930 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/989cae113946457a923fc3970bf32b1c is 50, key is test_row_0/C:col10/1731631871213/Put/seqid=0 2024-11-15T00:51:12,977 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742429_1605 (size=39949) 2024-11-15T00:51:12,977 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:12,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631932972, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,978 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:12,981 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=310, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/3807f1dd562648f5814c9414cf22356d 2024-11-15T00:51:12,988 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:12,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631932978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,988 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:12,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:12,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631932978, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:12,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742430_1606 (size=12949) 2024-11-15T00:51:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:12,992 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,003 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/ea64ac4d2c074ab582aa2f0ffbefcf84 is 50, key is test_row_0/B:col10/1731631871626/Put/seqid=0 2024-11-15T00:51:13,017 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742431_1607 (size=12301) 2024-11-15T00:51:13,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/ea64ac4d2c074ab582aa2f0ffbefcf84 2024-11-15T00:51:13,029 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/725a8d6d20b04263baa13e7edf7bf4a0 is 50, key is test_row_0/C:col10/1731631871626/Put/seqid=0 2024-11-15T00:51:13,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742432_1608 (size=12301) 2024-11-15T00:51:13,056 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=310 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/725a8d6d20b04263baa13e7edf7bf4a0 2024-11-15T00:51:13,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/3807f1dd562648f5814c9414cf22356d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/3807f1dd562648f5814c9414cf22356d 2024-11-15T00:51:13,067 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/3807f1dd562648f5814c9414cf22356d, entries=200, sequenceid=310, filesize=39.0 K 2024-11-15T00:51:13,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/ea64ac4d2c074ab582aa2f0ffbefcf84 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ea64ac4d2c074ab582aa2f0ffbefcf84 2024-11-15T00:51:13,077 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ea64ac4d2c074ab582aa2f0ffbefcf84, entries=150, sequenceid=310, filesize=12.0 K 2024-11-15T00:51:13,080 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/725a8d6d20b04263baa13e7edf7bf4a0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/725a8d6d20b04263baa13e7edf7bf4a0 2024-11-15T00:51:13,096 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/725a8d6d20b04263baa13e7edf7bf4a0, entries=150, sequenceid=310, filesize=12.0 K 2024-11-15T00:51:13,097 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 287ms, sequenceid=310, compaction requested=false 2024-11-15T00:51:13,097 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:13,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:13,192 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-15T00:51:13,192 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-15T00:51:13,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-15T00:51:13,201 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:51:13,201 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.3 flush TestAcidGuarantees 2024-11-15T00:51:13,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees 2024-11-15T00:51:13,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T00:51:13,208 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-15T00:51:13,209 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=166, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-15T00:51:13,209 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-15T00:51:13,235 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115700cd1aa231544d4bba43bca6e954699_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631873190/Put/seqid=0 2024-11-15T00:51:13,257 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#A#compaction#503 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:13,258 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/56a0bda2f169491bbd17610a7123b1d0 is 175, key is test_row_0/A:col10/1731631871213/Put/seqid=0 2024-11-15T00:51:13,271 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742433_1609 (size=14994) 2024-11-15T00:51:13,272 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742434_1610 (size=31903) 2024-11-15T00:51:13,281 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/56a0bda2f169491bbd17610a7123b1d0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/56a0bda2f169491bbd17610a7123b1d0 2024-11-15T00:51:13,283 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631933275, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,284 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631933276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,284 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,286 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/A of 4c1752556c034b2ca5cdf7bf0db8cd94 into 56a0bda2f169491bbd17610a7123b1d0(size=31.2 K), total size for store is 70.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:13,286 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:13,286 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/A, priority=13, startTime=1731631872697; duration=0sec 2024-11-15T00:51:13,287 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:13,287 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:A 2024-11-15T00:51:13,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 141 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631933282, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,287 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T00:51:13,363 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T00:51:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:13,364 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,364 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,390 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/989cae113946457a923fc3970bf32b1c as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/989cae113946457a923fc3970bf32b1c 2024-11-15T00:51:13,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631933386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,393 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,393 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631933387, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,394 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,396 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/C of 4c1752556c034b2ca5cdf7bf0db8cd94 into 989cae113946457a923fc3970bf32b1c(size=12.6 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:13,396 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:13,396 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/C, priority=13, startTime=1731631872697; duration=0sec 2024-11-15T00:51:13,396 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:13,396 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:C 2024-11-15T00:51:13,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 142 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631933401, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,412 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T00:51:13,516 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T00:51:13,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:13,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:13,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:13,517 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,517 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,598 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631933596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,599 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 133 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631933607, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,608 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 143 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631933617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,620 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,669 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T00:51:13,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:13,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:13,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:13,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,672 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:13,675 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115700cd1aa231544d4bba43bca6e954699_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115700cd1aa231544d4bba43bca6e954699_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:13,676 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/a0255a5d5a4a4f0fa055551e052c8892, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:13,676 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/a0255a5d5a4a4f0fa055551e052c8892 is 175, key is test_row_0/A:col10/1731631873190/Put/seqid=0 2024-11-15T00:51:13,689 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742435_1611 (size=39949) 2024-11-15T00:51:13,821 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T00:51:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:13,821 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:13,821 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T00:51:13,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631933906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,921 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631933916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,921 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,930 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:13,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 144 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631933927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:13,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:13,931 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:13,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T00:51:13,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:13,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:13,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:13,974 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:13,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:14,026 INFO [Thread-2361 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2361. 2024-11-15T00:51:14,026 DEBUG [Thread-2361 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:51:14,026 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:14,026 DEBUG [Thread-2361 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:14,026 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:14,027 INFO [Thread-2365 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2365. 2024-11-15T00:51:14,027 DEBUG [Thread-2365 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:51:14,027 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:14,027 DEBUG [Thread-2365 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:14,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:14,027 INFO [Thread-2359 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2359. 2024-11-15T00:51:14,027 DEBUG [Thread-2359 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:51:14,027 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:14,027 DEBUG [Thread-2359 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:14,027 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:14,030 INFO [Thread-2367 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2367. 2024-11-15T00:51:14,030 DEBUG [Thread-2367 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:51:14,030 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:14,030 DEBUG [Thread-2367 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:14,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:14,030 INFO [Thread-2363 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2363. 2024-11-15T00:51:14,030 DEBUG [Thread-2363 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:51:14,030 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:14,030 DEBUG [Thread-2363 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:14,030 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:14,089 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=324, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/a0255a5d5a4a4f0fa055551e052c8892 2024-11-15T00:51:14,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/448303d10d6e4885a5fe8d06e518d015 is 50, key is test_row_0/B:col10/1731631873190/Put/seqid=0 2024-11-15T00:51:14,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742436_1612 (size=12301) 2024-11-15T00:51:14,127 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T00:51:14,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:14,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,127 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:14,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:14,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:14,280 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T00:51:14,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:14,280 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,280 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:14,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-0 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:14,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:14,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T00:51:14,418 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:14,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33592 deadline: 1731631934418, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:14,419 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:14,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:14,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33682 deadline: 1731631934427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:14,427 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:14,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:14,428 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:14,433 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T00:51:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,433 ERROR [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] handler.RSProcedureHandler(58): pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:14,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-1 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=167 java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:14,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4177): Remote procedure failed, pid=167 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-15T00:51:14,437 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-15T00:51:14,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] ipc.CallRunner(138): callId: 145 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.3:33614 deadline: 1731631934437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 , the old value is region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-15T00:51:14,438 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., hostname=fb2f6cb7846a,42925,1731631698709, seqNum=5 because the exception is null or not the one we care about 2024-11-15T00:51:14,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/448303d10d6e4885a5fe8d06e518d015 2024-11-15T00:51:14,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/e601264ce3c24f6d95ced561753979f4 is 50, key is test_row_0/C:col10/1731631873190/Put/seqid=0 2024-11-15T00:51:14,518 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742437_1613 (size=12301) 2024-11-15T00:51:14,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/e601264ce3c24f6d95ced561753979f4 2024-11-15T00:51:14,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/a0255a5d5a4a4f0fa055551e052c8892 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a0255a5d5a4a4f0fa055551e052c8892 2024-11-15T00:51:14,526 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a0255a5d5a4a4f0fa055551e052c8892, entries=200, sequenceid=324, filesize=39.0 K 2024-11-15T00:51:14,526 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/448303d10d6e4885a5fe8d06e518d015 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/448303d10d6e4885a5fe8d06e518d015 2024-11-15T00:51:14,530 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/448303d10d6e4885a5fe8d06e518d015, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T00:51:14,531 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/e601264ce3c24f6d95ced561753979f4 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/e601264ce3c24f6d95ced561753979f4 2024-11-15T00:51:14,535 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/e601264ce3c24f6d95ced561753979f4, entries=150, sequenceid=324, filesize=12.0 K 2024-11-15T00:51:14,536 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1343ms, sequenceid=324, compaction requested=true 2024-11-15T00:51:14,536 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:14,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:A, priority=-2147483648, current under compaction store size is 1 2024-11-15T00:51:14,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:14,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:B, priority=-2147483648, current under compaction store size is 2 2024-11-15T00:51:14,536 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:51:14,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:14,536 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:51:14,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 4c1752556c034b2ca5cdf7bf0db8cd94:C, priority=-2147483648, current under compaction store size is 3 2024-11-15T00:51:14,536 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:51:14,537 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:51:14,537 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/B is initiating minor compaction (all files) 2024-11-15T00:51:14,537 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/B in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,537 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8d8ba2ff8b1c4eb4a3c14e0acb2bc996, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ea64ac4d2c074ab582aa2f0ffbefcf84, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/448303d10d6e4885a5fe8d06e518d015] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=36.7 K 2024-11-15T00:51:14,538 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111801 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:51:14,538 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/A is initiating minor compaction (all files) 2024-11-15T00:51:14,538 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/A in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,538 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/56a0bda2f169491bbd17610a7123b1d0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/3807f1dd562648f5814c9414cf22356d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a0255a5d5a4a4f0fa055551e052c8892] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=109.2 K 2024-11-15T00:51:14,538 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 8d8ba2ff8b1c4eb4a3c14e0acb2bc996, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731631871213 2024-11-15T00:51:14,538 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,538 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. files: [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/56a0bda2f169491bbd17610a7123b1d0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/3807f1dd562648f5814c9414cf22356d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a0255a5d5a4a4f0fa055551e052c8892] 2024-11-15T00:51:14,538 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting ea64ac4d2c074ab582aa2f0ffbefcf84, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731631871626 2024-11-15T00:51:14,538 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 56a0bda2f169491bbd17610a7123b1d0, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731631871213 2024-11-15T00:51:14,538 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3807f1dd562648f5814c9414cf22356d, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731631871612 2024-11-15T00:51:14,538 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 448303d10d6e4885a5fe8d06e518d015, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731631872838 2024-11-15T00:51:14,539 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0255a5d5a4a4f0fa055551e052c8892, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731631872838 2024-11-15T00:51:14,545 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:14,547 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#B#compaction#512 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:14,547 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/f3a6a71f1f034fafa262ff282bdc54b6 is 50, key is test_row_0/B:col10/1731631873190/Put/seqid=0 2024-11-15T00:51:14,549 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024111598c1a81c55494ae5817e8848f36bfd0f_4c1752556c034b2ca5cdf7bf0db8cd94 store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:14,553 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024111598c1a81c55494ae5817e8848f36bfd0f_4c1752556c034b2ca5cdf7bf0db8cd94, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:14,553 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024111598c1a81c55494ae5817e8848f36bfd0f_4c1752556c034b2ca5cdf7bf0db8cd94 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:14,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742438_1614 (size=13051) 2024-11-15T00:51:14,571 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/f3a6a71f1f034fafa262ff282bdc54b6 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/f3a6a71f1f034fafa262ff282bdc54b6 2024-11-15T00:51:14,583 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/B of 4c1752556c034b2ca5cdf7bf0db8cd94 into f3a6a71f1f034fafa262ff282bdc54b6(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:14,583 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:14,583 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/B, priority=13, startTime=1731631874536; duration=0sec 2024-11-15T00:51:14,583 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-15T00:51:14,583 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:B 2024-11-15T00:51:14,583 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-15T00:51:14,585 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742439_1615 (size=4469) 2024-11-15T00:51:14,585 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-15T00:51:14,585 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1541): 4c1752556c034b2ca5cdf7bf0db8cd94/C is initiating minor compaction (all files) 2024-11-15T00:51:14,585 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 4c1752556c034b2ca5cdf7bf0db8cd94/C in TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,585 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/989cae113946457a923fc3970bf32b1c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/725a8d6d20b04263baa13e7edf7bf4a0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/e601264ce3c24f6d95ced561753979f4] into tmpdir=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp, totalSize=36.7 K 2024-11-15T00:51:14,588 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 989cae113946457a923fc3970bf32b1c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=286, earliestPutTs=1731631871213 2024-11-15T00:51:14,589 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=42925 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=167 2024-11-15T00:51:14,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:14,589 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-15T00:51:14,592 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting 725a8d6d20b04263baa13e7edf7bf4a0, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=310, earliestPutTs=1731631871626 2024-11-15T00:51:14,600 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] compactions.Compactor(225): Compacting e601264ce3c24f6d95ced561753979f4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1731631872838 2024-11-15T00:51:14,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158105e7d8d1d3439f8cc5bfb3855e1cf3_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_0/A:col10/1731631873269/Put/seqid=0 2024-11-15T00:51:14,658 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#C#compaction#514 average throughput is 0.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:14,659 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/eb6caf637a444bdea1102b62d9e0a257 is 50, key is test_row_0/C:col10/1731631873190/Put/seqid=0 2024-11-15T00:51:14,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742440_1616 (size=12454) 2024-11-15T00:51:14,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742441_1617 (size=13051) 2024-11-15T00:51:14,682 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/eb6caf637a444bdea1102b62d9e0a257 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/eb6caf637a444bdea1102b62d9e0a257 2024-11-15T00:51:14,687 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/C of 4c1752556c034b2ca5cdf7bf0db8cd94 into eb6caf637a444bdea1102b62d9e0a257(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:14,687 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:14,687 INFO [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/C, priority=13, startTime=1731631874536; duration=0sec 2024-11-15T00:51:14,687 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:14,687 DEBUG [RS:0;fb2f6cb7846a:42925-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:C 2024-11-15T00:51:14,991 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 4c1752556c034b2ca5cdf7bf0db8cd94#A#compaction#511 average throughput is 0.05 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-15T00:51:14,991 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/8901588d523245b9bbacb5ab00724f16 is 175, key is test_row_0/A:col10/1731631873190/Put/seqid=0 2024-11-15T00:51:15,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742442_1618 (size=32005) 2024-11-15T00:51:15,023 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/8901588d523245b9bbacb5ab00724f16 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/8901588d523245b9bbacb5ab00724f16 2024-11-15T00:51:15,030 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 4c1752556c034b2ca5cdf7bf0db8cd94/A of 4c1752556c034b2ca5cdf7bf0db8cd94 into 8901588d523245b9bbacb5ab00724f16(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-15T00:51:15,030 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:15,030 INFO [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., storeName=4c1752556c034b2ca5cdf7bf0db8cd94/A, priority=13, startTime=1731631874536; duration=0sec 2024-11-15T00:51:15,030 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-15T00:51:15,030 DEBUG [RS:0;fb2f6cb7846a:42925-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 4c1752556c034b2ca5cdf7bf0db8cd94:A 2024-11-15T00:51:15,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:15,070 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411158105e7d8d1d3439f8cc5bfb3855e1cf3_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158105e7d8d1d3439f8cc5bfb3855e1cf3_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:15,070 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/9fc7400d4763420f8e929239d708644d, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:15,071 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/9fc7400d4763420f8e929239d708644d is 175, key is test_row_0/A:col10/1731631873269/Put/seqid=0 2024-11-15T00:51:15,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742443_1619 (size=31255) 2024-11-15T00:51:15,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T00:51:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=42925 {}] regionserver.HRegion(8855): Flush requested on 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:15,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. as already flushing 2024-11-15T00:51:15,428 INFO [Thread-2356 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2356. 2024-11-15T00:51:15,428 DEBUG [Thread-2356 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:51:15,428 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:15,428 DEBUG [Thread-2356 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:15,428 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:15,438 INFO [Thread-2348 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2348. 2024-11-15T00:51:15,438 DEBUG [Thread-2348 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:51:15,438 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:15,438 DEBUG [Thread-2348 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:15,438 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:15,449 INFO [Thread-2350 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2350. 2024-11-15T00:51:15,449 DEBUG [Thread-2350 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:51:15,449 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:15,449 DEBUG [Thread-2350 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:15,449 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:15,475 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=350, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/9fc7400d4763420f8e929239d708644d 2024-11-15T00:51:15,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/711b8b558e374e3d897f209cc150795a is 50, key is test_row_0/B:col10/1731631873269/Put/seqid=0 2024-11-15T00:51:15,485 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742444_1620 (size=12301) 2024-11-15T00:51:15,886 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/711b8b558e374e3d897f209cc150795a 2024-11-15T00:51:15,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/4a946263cdfe423c8c82a5f26bdd9852 is 50, key is test_row_0/C:col10/1731631873269/Put/seqid=0 2024-11-15T00:51:15,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742445_1621 (size=12301) 2024-11-15T00:51:16,307 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=350 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/4a946263cdfe423c8c82a5f26bdd9852 2024-11-15T00:51:16,338 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/9fc7400d4763420f8e929239d708644d as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/9fc7400d4763420f8e929239d708644d 2024-11-15T00:51:16,343 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/9fc7400d4763420f8e929239d708644d, entries=150, sequenceid=350, filesize=30.5 K 2024-11-15T00:51:16,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/711b8b558e374e3d897f209cc150795a as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/711b8b558e374e3d897f209cc150795a 2024-11-15T00:51:16,346 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/711b8b558e374e3d897f209cc150795a, entries=150, sequenceid=350, filesize=12.0 K 2024-11-15T00:51:16,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/4a946263cdfe423c8c82a5f26bdd9852 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4a946263cdfe423c8c82a5f26bdd9852 2024-11-15T00:51:16,350 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4a946263cdfe423c8c82a5f26bdd9852, entries=150, sequenceid=350, filesize=12.0 K 2024-11-15T00:51:16,351 INFO [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=20.13 KB/20610 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1762ms, sequenceid=350, compaction requested=false 2024-11-15T00:51:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.HRegion(2603): Flush status journal for 4c1752556c034b2ca5cdf7bf0db8cd94: 2024-11-15T00:51:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:16,351 DEBUG [RS_FLUSH_OPERATIONS-regionserver/fb2f6cb7846a:0-2 {event_type=RS_FLUSH_REGIONS, pid=167}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=167 2024-11-15T00:51:16,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster(4169): Remote procedure done, pid=167 2024-11-15T00:51:16,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-15T00:51:16,353 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.1430 sec 2024-11-15T00:51:16,355 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=166, table=TestAcidGuarantees in 3.1530 sec 2024-11-15T00:51:16,788 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-15T00:51:16,948 INFO [Thread-2354 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2354. 2024-11-15T00:51:16,948 DEBUG [Thread-2354 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:51:16,948 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:16,948 DEBUG [Thread-2354 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:16,948 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:17,010 INFO [Thread-2352 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2352. 2024-11-15T00:51:17,010 DEBUG [Thread-2352 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-15T00:51:17,010 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:17,011 DEBUG [Thread-2352 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:17,011 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:17,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=166 2024-11-15T00:51:17,367 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 75 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 85 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 22 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 19 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 80 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2503 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2519 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2517 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2517 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2552 2024-11-15T00:51:17,367 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-15T00:51:17,367 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:51:17,367 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:51:17,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:17,367 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:17,367 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:17,368 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.3 disable TestAcidGuarantees 2024-11-15T00:51:17,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-15T00:51:17,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T00:51:17,371 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631877371"}]},"ts":"1731631877371"} 2024-11-15T00:51:17,373 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-15T00:51:17,374 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-15T00:51:17,374 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=169, ppid=168, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-15T00:51:17,376 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, UNASSIGN}] 2024-11-15T00:51:17,377 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, UNASSIGN 2024-11-15T00:51:17,377 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=4c1752556c034b2ca5cdf7bf0db8cd94, regionState=CLOSING, regionLocation=fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:17,378 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=170, ppid=169, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, UNASSIGN because future has completed 2024-11-15T00:51:17,379 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-15T00:51:17,379 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709}] 2024-11-15T00:51:17,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T00:51:17,531 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(122): Close 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:17,531 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-15T00:51:17,531 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1722): Closing 4c1752556c034b2ca5cdf7bf0db8cd94, disabling compactions & flushes 2024-11-15T00:51:17,531 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:17,531 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:17,531 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. after waiting 0 ms 2024-11-15T00:51:17,531 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:17,531 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(2902): Flushing 4c1752556c034b2ca5cdf7bf0db8cd94 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-15T00:51:17,537 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b9604958b07649449ca91449939c9174_4c1752556c034b2ca5cdf7bf0db8cd94 is 50, key is test_row_1/A:col10/1731631876947/Put/seqid=0 2024-11-15T00:51:17,542 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742446_1622 (size=9914) 2024-11-15T00:51:17,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T00:51:17,942 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-15T00:51:17,946 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241115b9604958b07649449ca91449939c9174_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b9604958b07649449ca91449939c9174_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:17,947 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/19732f6d018642e58d1d65736e4d5731, store: [table=TestAcidGuarantees family=A region=4c1752556c034b2ca5cdf7bf0db8cd94] 2024-11-15T00:51:17,947 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/19732f6d018642e58d1d65736e4d5731 is 175, key is test_row_1/A:col10/1731631876947/Put/seqid=0 2024-11-15T00:51:17,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742447_1623 (size=22561) 2024-11-15T00:51:17,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T00:51:18,353 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=360, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/19732f6d018642e58d1d65736e4d5731 2024-11-15T00:51:18,359 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/c8280592d1124f40b1d6bd17bb7308ae is 50, key is test_row_1/B:col10/1731631876947/Put/seqid=0 2024-11-15T00:51:18,362 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742448_1624 (size=9857) 2024-11-15T00:51:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T00:51:18,763 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/c8280592d1124f40b1d6bd17bb7308ae 2024-11-15T00:51:18,768 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/28bfbdebca8a4f68b9dd5c1992dd6809 is 50, key is test_row_1/C:col10/1731631876947/Put/seqid=0 2024-11-15T00:51:18,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742449_1625 (size=9857) 2024-11-15T00:51:19,172 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=360 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/28bfbdebca8a4f68b9dd5c1992dd6809 2024-11-15T00:51:19,176 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/A/19732f6d018642e58d1d65736e4d5731 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/19732f6d018642e58d1d65736e4d5731 2024-11-15T00:51:19,180 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/19732f6d018642e58d1d65736e4d5731, entries=100, sequenceid=360, filesize=22.0 K 2024-11-15T00:51:19,181 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/B/c8280592d1124f40b1d6bd17bb7308ae as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/c8280592d1124f40b1d6bd17bb7308ae 2024-11-15T00:51:19,184 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/c8280592d1124f40b1d6bd17bb7308ae, entries=100, sequenceid=360, filesize=9.6 K 2024-11-15T00:51:19,184 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/.tmp/C/28bfbdebca8a4f68b9dd5c1992dd6809 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/28bfbdebca8a4f68b9dd5c1992dd6809 2024-11-15T00:51:19,187 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/28bfbdebca8a4f68b9dd5c1992dd6809, entries=100, sequenceid=360, filesize=9.6 K 2024-11-15T00:51:19,188 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1657ms, sequenceid=360, compaction requested=true 2024-11-15T00:51:19,188 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/bb25094849a940f4aada0393fbd0bd73, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f19d2438629543918a181e51377a7f7d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/b388374d17964cc7a5d294230f5889a3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/48eaa8142e1242489e25f24eca5ee9d8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/70c1c743ea5448d2a5e47d6e5e7247ac, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/29ce6c68d5ae4e00a31e78fc93d0327a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/499d2dc1c4b84a988e271f00c3c4160e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f8a10eb4311144be848b621c0eb4be26, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/c41d71974c7e43778f9226e5ea2a5e7d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/d46d35a6472c4624aba30ae573469904, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/e52de19f37b2472394449e24136d16c3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/8bcece740f6d46b899a32cf070421c76, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/0bdd92d9797e44b8bdc8d169301532d8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/106b42f8190c4f7084d6b0813af66002, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/893dae7e68414410b266362324f769de, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/63e12e73e8484a1b93ee88ef57ee97bb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a8c5870408074a558bd7fd2448e07d80, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/896927db12d5456589d9f515cb3adbcb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/56a0bda2f169491bbd17610a7123b1d0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/7d50ceb8610042859712213af74f8b21, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/3807f1dd562648f5814c9414cf22356d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a0255a5d5a4a4f0fa055551e052c8892] to archive 2024-11-15T00:51:19,189 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:51:19,191 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/bb25094849a940f4aada0393fbd0bd73 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/bb25094849a940f4aada0393fbd0bd73 2024-11-15T00:51:19,192 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f19d2438629543918a181e51377a7f7d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f19d2438629543918a181e51377a7f7d 2024-11-15T00:51:19,193 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/b388374d17964cc7a5d294230f5889a3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/b388374d17964cc7a5d294230f5889a3 2024-11-15T00:51:19,194 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/48eaa8142e1242489e25f24eca5ee9d8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/48eaa8142e1242489e25f24eca5ee9d8 2024-11-15T00:51:19,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/70c1c743ea5448d2a5e47d6e5e7247ac to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/70c1c743ea5448d2a5e47d6e5e7247ac 2024-11-15T00:51:19,195 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/29ce6c68d5ae4e00a31e78fc93d0327a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/29ce6c68d5ae4e00a31e78fc93d0327a 2024-11-15T00:51:19,196 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/499d2dc1c4b84a988e271f00c3c4160e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/499d2dc1c4b84a988e271f00c3c4160e 2024-11-15T00:51:19,197 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f8a10eb4311144be848b621c0eb4be26 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/f8a10eb4311144be848b621c0eb4be26 2024-11-15T00:51:19,198 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/c41d71974c7e43778f9226e5ea2a5e7d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/c41d71974c7e43778f9226e5ea2a5e7d 2024-11-15T00:51:19,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/d46d35a6472c4624aba30ae573469904 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/d46d35a6472c4624aba30ae573469904 2024-11-15T00:51:19,199 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/e52de19f37b2472394449e24136d16c3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/e52de19f37b2472394449e24136d16c3 2024-11-15T00:51:19,200 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/8bcece740f6d46b899a32cf070421c76 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/8bcece740f6d46b899a32cf070421c76 2024-11-15T00:51:19,201 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/0bdd92d9797e44b8bdc8d169301532d8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/0bdd92d9797e44b8bdc8d169301532d8 2024-11-15T00:51:19,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/106b42f8190c4f7084d6b0813af66002 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/106b42f8190c4f7084d6b0813af66002 2024-11-15T00:51:19,202 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/893dae7e68414410b266362324f769de to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/893dae7e68414410b266362324f769de 2024-11-15T00:51:19,203 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/63e12e73e8484a1b93ee88ef57ee97bb to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/63e12e73e8484a1b93ee88ef57ee97bb 2024-11-15T00:51:19,204 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a8c5870408074a558bd7fd2448e07d80 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a8c5870408074a558bd7fd2448e07d80 2024-11-15T00:51:19,205 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/896927db12d5456589d9f515cb3adbcb to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/896927db12d5456589d9f515cb3adbcb 2024-11-15T00:51:19,206 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/56a0bda2f169491bbd17610a7123b1d0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/56a0bda2f169491bbd17610a7123b1d0 2024-11-15T00:51:19,207 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/7d50ceb8610042859712213af74f8b21 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/7d50ceb8610042859712213af74f8b21 2024-11-15T00:51:19,208 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/3807f1dd562648f5814c9414cf22356d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/3807f1dd562648f5814c9414cf22356d 2024-11-15T00:51:19,209 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a0255a5d5a4a4f0fa055551e052c8892 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/a0255a5d5a4a4f0fa055551e052c8892 2024-11-15T00:51:19,216 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/7f14a1fc575a4eb2af9dcb878c5e94e1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/1e6aef98e7624870ab6a9dee0e711acf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ce66d56c72674cb791b753d302b58bca, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/0d384dbb695e412ebf4e36a187d8bdd9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/56b11721848b46ba9e3c0a67cd03c523, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/5cb131d640f844ba913cda004d30744d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/627b05101ece44a6be5e5f582e1a008e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8531e99fa6044f4aac633f0901ea005a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/53d39b4fcc634d37b146c48b780ee9d0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/490da2fc561c4d94b175c7f55039869c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ddcf1208e1014a39a09a307a927088d4, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/26fd1e4ae51a469faf81e4f11cded4d3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/fdaca34fe5784f0487bdf55e28d18cfa, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/522fe394e52c49cda53f0b0de1886c75, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/a1b9508ff81346048dd7cbba570b14f1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8da5cb5f7151449c800d94691b38d18f, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/13fc7a90657a463e8b8e74f8629116cb, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/33a571d501b440338836188f65b282a3, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8d8ba2ff8b1c4eb4a3c14e0acb2bc996, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/66c174f59b034f01943383c81e8659a9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ea64ac4d2c074ab582aa2f0ffbefcf84, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/448303d10d6e4885a5fe8d06e518d015] to archive 2024-11-15T00:51:19,217 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:51:19,219 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/7f14a1fc575a4eb2af9dcb878c5e94e1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/7f14a1fc575a4eb2af9dcb878c5e94e1 2024-11-15T00:51:19,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/1e6aef98e7624870ab6a9dee0e711acf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/1e6aef98e7624870ab6a9dee0e711acf 2024-11-15T00:51:19,220 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ce66d56c72674cb791b753d302b58bca to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ce66d56c72674cb791b753d302b58bca 2024-11-15T00:51:19,221 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/0d384dbb695e412ebf4e36a187d8bdd9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/0d384dbb695e412ebf4e36a187d8bdd9 2024-11-15T00:51:19,222 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/56b11721848b46ba9e3c0a67cd03c523 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/56b11721848b46ba9e3c0a67cd03c523 2024-11-15T00:51:19,223 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/5cb131d640f844ba913cda004d30744d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/5cb131d640f844ba913cda004d30744d 2024-11-15T00:51:19,224 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/627b05101ece44a6be5e5f582e1a008e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/627b05101ece44a6be5e5f582e1a008e 2024-11-15T00:51:19,225 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8531e99fa6044f4aac633f0901ea005a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8531e99fa6044f4aac633f0901ea005a 2024-11-15T00:51:19,227 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/53d39b4fcc634d37b146c48b780ee9d0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/53d39b4fcc634d37b146c48b780ee9d0 2024-11-15T00:51:19,228 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/490da2fc561c4d94b175c7f55039869c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/490da2fc561c4d94b175c7f55039869c 2024-11-15T00:51:19,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ddcf1208e1014a39a09a307a927088d4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ddcf1208e1014a39a09a307a927088d4 2024-11-15T00:51:19,229 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/26fd1e4ae51a469faf81e4f11cded4d3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/26fd1e4ae51a469faf81e4f11cded4d3 2024-11-15T00:51:19,230 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/fdaca34fe5784f0487bdf55e28d18cfa to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/fdaca34fe5784f0487bdf55e28d18cfa 2024-11-15T00:51:19,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/522fe394e52c49cda53f0b0de1886c75 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/522fe394e52c49cda53f0b0de1886c75 2024-11-15T00:51:19,231 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/a1b9508ff81346048dd7cbba570b14f1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/a1b9508ff81346048dd7cbba570b14f1 2024-11-15T00:51:19,233 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8da5cb5f7151449c800d94691b38d18f to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8da5cb5f7151449c800d94691b38d18f 2024-11-15T00:51:19,234 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/13fc7a90657a463e8b8e74f8629116cb to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/13fc7a90657a463e8b8e74f8629116cb 2024-11-15T00:51:19,235 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/33a571d501b440338836188f65b282a3 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/33a571d501b440338836188f65b282a3 2024-11-15T00:51:19,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8d8ba2ff8b1c4eb4a3c14e0acb2bc996 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/8d8ba2ff8b1c4eb4a3c14e0acb2bc996 2024-11-15T00:51:19,236 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/66c174f59b034f01943383c81e8659a9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/66c174f59b034f01943383c81e8659a9 2024-11-15T00:51:19,237 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ea64ac4d2c074ab582aa2f0ffbefcf84 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/ea64ac4d2c074ab582aa2f0ffbefcf84 2024-11-15T00:51:19,238 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/448303d10d6e4885a5fe8d06e518d015 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/448303d10d6e4885a5fe8d06e518d015 2024-11-15T00:51:19,239 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/cbe146b45046416384af802e506fe4f8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2b622549d1cc429d93e6b6298ad592e1, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4beb458d261c4a6198877f0b03738ae8, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/0f7d60f8e2954498834beb9251cc1da9, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/86683b2c27834ba59cf460522f629adc, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f6dd7317f3b54d4c846e86f4219fc6ab, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a318135540bf4b00a9358d0c2280b8c5, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4adda3196d804f89953e37cf8d9ea678, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/dd14a784626d4497943f470fbeb42dce, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/00d46966bb614bdaa3ebf1838075b4fd, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/daea7fcc67774a7b9c221d1cbca8afdf, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b3764ba45bad4430bbda5db81cc02dce, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b7df6ad5b76d4cb29c6975e648b23c84, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2174c2580e32404ba9f82a22234e994e, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a3a3cd0bf23c46afbb958571fd19b491, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/16b6a0d510bf4f048afafed079dac53d, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/1488332f5a5b4efaac4886e640dba11b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f825d5503d554297af2fbe720b84df6a, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/989cae113946457a923fc3970bf32b1c, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/80a694e251c440f9b5e584dc469a247b, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/725a8d6d20b04263baa13e7edf7bf4a0, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/e601264ce3c24f6d95ced561753979f4] to archive 2024-11-15T00:51:19,240 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-15T00:51:19,241 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/cbe146b45046416384af802e506fe4f8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/cbe146b45046416384af802e506fe4f8 2024-11-15T00:51:19,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2b622549d1cc429d93e6b6298ad592e1 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2b622549d1cc429d93e6b6298ad592e1 2024-11-15T00:51:19,242 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4beb458d261c4a6198877f0b03738ae8 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4beb458d261c4a6198877f0b03738ae8 2024-11-15T00:51:19,243 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/0f7d60f8e2954498834beb9251cc1da9 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/0f7d60f8e2954498834beb9251cc1da9 2024-11-15T00:51:19,244 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/86683b2c27834ba59cf460522f629adc to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/86683b2c27834ba59cf460522f629adc 2024-11-15T00:51:19,245 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f6dd7317f3b54d4c846e86f4219fc6ab to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f6dd7317f3b54d4c846e86f4219fc6ab 2024-11-15T00:51:19,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a318135540bf4b00a9358d0c2280b8c5 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a318135540bf4b00a9358d0c2280b8c5 2024-11-15T00:51:19,246 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4adda3196d804f89953e37cf8d9ea678 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4adda3196d804f89953e37cf8d9ea678 2024-11-15T00:51:19,247 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/dd14a784626d4497943f470fbeb42dce to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/dd14a784626d4497943f470fbeb42dce 2024-11-15T00:51:19,248 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/00d46966bb614bdaa3ebf1838075b4fd to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/00d46966bb614bdaa3ebf1838075b4fd 2024-11-15T00:51:19,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/daea7fcc67774a7b9c221d1cbca8afdf to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/daea7fcc67774a7b9c221d1cbca8afdf 2024-11-15T00:51:19,249 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b3764ba45bad4430bbda5db81cc02dce to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b3764ba45bad4430bbda5db81cc02dce 2024-11-15T00:51:19,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b7df6ad5b76d4cb29c6975e648b23c84 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/b7df6ad5b76d4cb29c6975e648b23c84 2024-11-15T00:51:19,250 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2174c2580e32404ba9f82a22234e994e to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/2174c2580e32404ba9f82a22234e994e 2024-11-15T00:51:19,251 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a3a3cd0bf23c46afbb958571fd19b491 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/a3a3cd0bf23c46afbb958571fd19b491 2024-11-15T00:51:19,252 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/16b6a0d510bf4f048afafed079dac53d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/16b6a0d510bf4f048afafed079dac53d 2024-11-15T00:51:19,253 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/1488332f5a5b4efaac4886e640dba11b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/1488332f5a5b4efaac4886e640dba11b 2024-11-15T00:51:19,254 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f825d5503d554297af2fbe720b84df6a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/f825d5503d554297af2fbe720b84df6a 2024-11-15T00:51:19,255 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/989cae113946457a923fc3970bf32b1c to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/989cae113946457a923fc3970bf32b1c 2024-11-15T00:51:19,256 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/80a694e251c440f9b5e584dc469a247b to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/80a694e251c440f9b5e584dc469a247b 2024-11-15T00:51:19,257 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/725a8d6d20b04263baa13e7edf7bf4a0 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/725a8d6d20b04263baa13e7edf7bf4a0 2024-11-15T00:51:19,258 DEBUG [StoreCloser-TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/e601264ce3c24f6d95ced561753979f4 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/e601264ce3c24f6d95ced561753979f4 2024-11-15T00:51:19,262 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/recovered.edits/363.seqid, newMaxSeqId=363, maxSeqId=4 2024-11-15T00:51:19,262 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. 2024-11-15T00:51:19,262 DEBUG [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] regionserver.HRegion(1676): Region close journal for 4c1752556c034b2ca5cdf7bf0db8cd94: Waiting for close lock at 1731631877531Running coprocessor pre-close hooks at 1731631877531Disabling compacts and flushes for region at 1731631877531Disabling writes for close at 1731631877531Obtaining lock to block concurrent updates at 1731631877531Preparing flush snapshotting stores in 4c1752556c034b2ca5cdf7bf0db8cd94 at 1731631877531Finished memstore snapshotting TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1731631877532 (+1 ms)Flushing stores of TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94. at 1731631877532Flushing 4c1752556c034b2ca5cdf7bf0db8cd94/A: creating writer at 1731631877532Flushing mob file 4c1752556c034b2ca5cdf7bf0db8cd94/A: appending metadata at 1731631877537 (+5 ms)Flushing mob file 4c1752556c034b2ca5cdf7bf0db8cd94/A: closing flushed file at 1731631877537Flushing 4c1752556c034b2ca5cdf7bf0db8cd94/A: appending metadata at 1731631877947 (+410 ms)Flushing 4c1752556c034b2ca5cdf7bf0db8cd94/A: closing flushed file at 1731631877947Flushing 4c1752556c034b2ca5cdf7bf0db8cd94/B: creating writer at 1731631878356 (+409 ms)Flushing 4c1752556c034b2ca5cdf7bf0db8cd94/B: appending metadata at 1731631878358 (+2 ms)Flushing 4c1752556c034b2ca5cdf7bf0db8cd94/B: closing flushed file at 1731631878358Flushing 4c1752556c034b2ca5cdf7bf0db8cd94/C: creating writer at 1731631878766 (+408 ms)Flushing 4c1752556c034b2ca5cdf7bf0db8cd94/C: appending metadata at 1731631878768 (+2 ms)Flushing 4c1752556c034b2ca5cdf7bf0db8cd94/C: closing flushed file at 1731631878768Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4e5ee8ff: reopening flushed file at 1731631879175 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@c639717: reopening flushed file at 1731631879180 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@744c5f1c: reopening flushed file at 1731631879184 (+4 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 4c1752556c034b2ca5cdf7bf0db8cd94 in 1657ms, sequenceid=360, compaction requested=true at 1731631879188 (+4 ms)Writing region close event to WAL at 1731631879259 (+71 ms)Running coprocessor post-close hooks at 1731631879262 (+3 ms)Closed at 1731631879262 2024-11-15T00:51:19,264 INFO [RS_CLOSE_REGION-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_REGION, pid=171}] handler.UnassignRegionHandler(157): Closed 4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,265 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=170 updating hbase:meta row=4c1752556c034b2ca5cdf7bf0db8cd94, regionState=CLOSED 2024-11-15T00:51:19,266 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=171, ppid=170, state=RUNNABLE, hasLock=false; CloseRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 because future has completed 2024-11-15T00:51:19,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=171, resume processing ppid=170 2024-11-15T00:51:19,270 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=171, ppid=170, state=SUCCESS, hasLock=false; CloseRegionProcedure 4c1752556c034b2ca5cdf7bf0db8cd94, server=fb2f6cb7846a,42925,1731631698709 in 1.8880 sec 2024-11-15T00:51:19,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=170, resume processing ppid=169 2024-11-15T00:51:19,271 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=170, ppid=169, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=4c1752556c034b2ca5cdf7bf0db8cd94, UNASSIGN in 1.8940 sec 2024-11-15T00:51:19,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=169, resume processing ppid=168 2024-11-15T00:51:19,273 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=169, ppid=168, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.8980 sec 2024-11-15T00:51:19,274 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1731631879274"}]},"ts":"1731631879274"} 2024-11-15T00:51:19,276 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-15T00:51:19,276 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-15T00:51:19,277 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9090 sec 2024-11-15T00:51:19,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-15T00:51:19,517 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:51:19,518 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.3 delete TestAcidGuarantees 2024-11-15T00:51:19,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] procedure2.ProcedureExecutor(1139): Stored pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:51:19,519 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=172, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:51:19,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T00:51:19,520 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=172, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:51:19,522 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,524 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C, FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/recovered.edits] 2024-11-15T00:51:19,526 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/19732f6d018642e58d1d65736e4d5731 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/19732f6d018642e58d1d65736e4d5731 2024-11-15T00:51:19,526 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/8901588d523245b9bbacb5ab00724f16 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/8901588d523245b9bbacb5ab00724f16 2024-11-15T00:51:19,527 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/9fc7400d4763420f8e929239d708644d to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/A/9fc7400d4763420f8e929239d708644d 2024-11-15T00:51:19,528 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/711b8b558e374e3d897f209cc150795a to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/711b8b558e374e3d897f209cc150795a 2024-11-15T00:51:19,529 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/c8280592d1124f40b1d6bd17bb7308ae to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/c8280592d1124f40b1d6bd17bb7308ae 2024-11-15T00:51:19,529 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/f3a6a71f1f034fafa262ff282bdc54b6 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/B/f3a6a71f1f034fafa262ff282bdc54b6 2024-11-15T00:51:19,531 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/28bfbdebca8a4f68b9dd5c1992dd6809 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/28bfbdebca8a4f68b9dd5c1992dd6809 2024-11-15T00:51:19,531 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4a946263cdfe423c8c82a5f26bdd9852 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/4a946263cdfe423c8c82a5f26bdd9852 2024-11-15T00:51:19,532 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/eb6caf637a444bdea1102b62d9e0a257 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/C/eb6caf637a444bdea1102b62d9e0a257 2024-11-15T00:51:19,534 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/recovered.edits/363.seqid to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94/recovered.edits/363.seqid 2024-11-15T00:51:19,534 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/default/TestAcidGuarantees/4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,534 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-15T00:51:19,534 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:51:19,535 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-15T00:51:19,537 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111503ad76cf7d0e48fdb641d68f933b2968_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111503ad76cf7d0e48fdb641d68f933b2968_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,538 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111506cb38e086614d47b2d94d663b48042b_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111506cb38e086614d47b2d94d663b48042b_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,538 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115312f9f09152144d89c499a2cde3726a5_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115312f9f09152144d89c499a2cde3726a5_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,539 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111543512be5fb034459b1536e9077266406_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024111543512be5fb034459b1536e9077266406_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,540 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115700cd1aa231544d4bba43bca6e954699_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115700cd1aa231544d4bba43bca6e954699_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,540 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115781e25b4fc874ece8b7cba80f347e493_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115781e25b4fc874ece8b7cba80f347e493_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,541 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158105e7d8d1d3439f8cc5bfb3855e1cf3_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158105e7d8d1d3439f8cc5bfb3855e1cf3_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,542 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158daa04502444456895bea03064d497bb_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411158daa04502444456895bea03064d497bb_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,543 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b9604958b07649449ca91449939c9174_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115b9604958b07649449ca91449939c9174_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,543 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf8552c51d5740e6a27bd614b15652a4_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115bf8552c51d5740e6a27bd614b15652a4_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,544 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c15cd4f0e97945d087c6455b79228a41_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c15cd4f0e97945d087c6455b79228a41_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,545 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c1aecd4d789948cbb0d19ca630506c5d_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c1aecd4d789948cbb0d19ca630506c5d_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,545 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c9b9f68d77064774b5533d5f3c35eec0_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115c9b9f68d77064774b5533d5f3c35eec0_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,546 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ca915061b9964701ab260f97bc2e624e_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115ca915061b9964701ab260f97bc2e624e_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,547 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cac801f9dad847ae8497adc111d64490_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115cac801f9dad847ae8497adc111d64490_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,547 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dadacaa2306b41e69ec5e1377e0d8f40_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115dadacaa2306b41e69ec5e1377e0d8f40_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,548 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e6780a8b72254f849e901ec51b455888_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115e6780a8b72254f849e901ec51b455888_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,549 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f304bb28f6c344e988279add2c3801bd_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f304bb28f6c344e988279add2c3801bd_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,549 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f6a75ff6c27049f3a912e03e22a96220_4c1752556c034b2ca5cdf7bf0db8cd94 to hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241115f6a75ff6c27049f3a912e03e22a96220_4c1752556c034b2ca5cdf7bf0db8cd94 2024-11-15T00:51:19,550 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-15T00:51:19,552 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=172, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:51:19,553 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-15T00:51:19,555 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-15T00:51:19,556 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=172, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:51:19,556 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-15T00:51:19,556 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1731631879556"}]},"ts":"9223372036854775807"} 2024-11-15T00:51:19,558 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-15T00:51:19,558 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 4c1752556c034b2ca5cdf7bf0db8cd94, NAME => 'TestAcidGuarantees,,1731631850847.4c1752556c034b2ca5cdf7bf0db8cd94.', STARTKEY => '', ENDKEY => ''}] 2024-11-15T00:51:19,558 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-15T00:51:19,558 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1731631879558"}]},"ts":"9223372036854775807"} 2024-11-15T00:51:19,560 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-15T00:51:19,560 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=172, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-15T00:51:19,561 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=172, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 42 msec 2024-11-15T00:51:19,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=46397 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=172 2024-11-15T00:51:19,627 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-15T00:51:19,627 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-15T00:51:19,636 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241 (was 236) - Thread LEAK? -, OpenFileDescriptor=457 (was 443) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=999 (was 1001), ProcessCount=11 (was 11), AvailableMemoryMB=1737 (was 2870) 2024-11-15T00:51:19,636 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-15T00:51:19,636 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:51:19,636 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:51:19,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:19,636 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:19,636 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-15T00:51:19,637 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-15T00:51:19,637 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=1945866401, stopped=false 2024-11-15T00:51:19,637 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=fb2f6cb7846a,46397,1731631697729 2024-11-15T00:51:19,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T00:51:19,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-15T00:51:19,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:51:19,708 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:51:19,709 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:51:19,709 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:51:19,709 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-15T00:51:19,709 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:51:19,709 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:19,710 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server 'fb2f6cb7846a,42925,1731631698709' ***** 2024-11-15T00:51:19,710 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-15T00:51:19,710 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-15T00:51:19,711 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-15T00:51:19,711 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-15T00:51:19,711 INFO [RS:0;fb2f6cb7846a:42925 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-15T00:51:19,711 INFO [RS:0;fb2f6cb7846a:42925 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-15T00:51:19,711 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(959): stopping server fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:19,712 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T00:51:19,712 INFO [RS:0;fb2f6cb7846a:42925 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;fb2f6cb7846a:42925. 2024-11-15T00:51:19,712 DEBUG [RS:0;fb2f6cb7846a:42925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-15T00:51:19,712 DEBUG [RS:0;fb2f6cb7846a:42925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:19,712 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-15T00:51:19,716 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-15T00:51:19,716 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-15T00:51:19,716 INFO [regionserver/fb2f6cb7846a:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T00:51:19,716 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-15T00:51:19,717 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-15T00:51:19,717 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-15T00:51:19,718 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-15T00:51:19,718 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:51:19,718 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-15T00:51:19,718 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-15T00:51:19,718 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-15T00:51:19,718 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-15T00:51:19,718 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-15T00:51:19,750 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/info/2bace3666d4f46f0883c1b097dc136ab is 95, key is TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20./info:/1731631729268/DeleteFamily/seqid=0 2024-11-15T00:51:19,769 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742450_1626 (size=6514) 2024-11-15T00:51:19,770 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/info/2bace3666d4f46f0883c1b097dc136ab 2024-11-15T00:51:19,797 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/ns/1a923dd96a6541d993f8975871618a3e is 93, key is TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20./ns:/1731631729208/DeleteFamily/seqid=0 2024-11-15T00:51:19,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742451_1627 (size=5906) 2024-11-15T00:51:19,805 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/ns/1a923dd96a6541d993f8975871618a3e 2024-11-15T00:51:19,832 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/rep_barrier/4db17f6531e2403d9a6827b7648e8eb3 is 102, key is TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20./rep_barrier:/1731631729208/DeleteFamily/seqid=0 2024-11-15T00:51:19,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742452_1628 (size=6025) 2024-11-15T00:51:19,918 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:51:20,118 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:51:20,241 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/rep_barrier/4db17f6531e2403d9a6827b7648e8eb3 2024-11-15T00:51:20,257 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/table/7fec73bf776c410f9012bba8338a88b0 is 96, key is TestAcidGuarantees,,1731631703021.5b99d5a4d9ee135a84ccb828f4037b20./table:/1731631729208/DeleteFamily/seqid=0 2024-11-15T00:51:20,260 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742453_1629 (size=5936) 2024-11-15T00:51:20,268 INFO [regionserver/fb2f6cb7846a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-15T00:51:20,268 INFO [regionserver/fb2f6cb7846a:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-15T00:51:20,318 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:51:20,519 DEBUG [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-15T00:51:20,661 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/table/7fec73bf776c410f9012bba8338a88b0 2024-11-15T00:51:20,665 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/info/2bace3666d4f46f0883c1b097dc136ab as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/info/2bace3666d4f46f0883c1b097dc136ab 2024-11-15T00:51:20,669 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/info/2bace3666d4f46f0883c1b097dc136ab, entries=12, sequenceid=90, filesize=6.4 K 2024-11-15T00:51:20,670 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/ns/1a923dd96a6541d993f8975871618a3e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/ns/1a923dd96a6541d993f8975871618a3e 2024-11-15T00:51:20,674 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/ns/1a923dd96a6541d993f8975871618a3e, entries=8, sequenceid=90, filesize=5.8 K 2024-11-15T00:51:20,675 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/rep_barrier/4db17f6531e2403d9a6827b7648e8eb3 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/rep_barrier/4db17f6531e2403d9a6827b7648e8eb3 2024-11-15T00:51:20,678 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/rep_barrier/4db17f6531e2403d9a6827b7648e8eb3, entries=6, sequenceid=90, filesize=5.9 K 2024-11-15T00:51:20,679 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/.tmp/table/7fec73bf776c410f9012bba8338a88b0 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/table/7fec73bf776c410f9012bba8338a88b0 2024-11-15T00:51:20,684 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/table/7fec73bf776c410f9012bba8338a88b0, entries=7, sequenceid=90, filesize=5.8 K 2024-11-15T00:51:20,684 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 966ms, sequenceid=90, compaction requested=false 2024-11-15T00:51:20,689 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-15T00:51:20,690 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-15T00:51:20,690 INFO [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-15T00:51:20,690 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1731631879717Running coprocessor pre-close hooks at 1731631879718 (+1 ms)Disabling compacts and flushes for region at 1731631879718Disabling writes for close at 1731631879718Obtaining lock to block concurrent updates at 1731631879718Preparing flush snapshotting stores in 1588230740 at 1731631879718Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1731631879719 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1731631879719Flushing 1588230740/info: creating writer at 1731631879720 (+1 ms)Flushing 1588230740/info: appending metadata at 1731631879750 (+30 ms)Flushing 1588230740/info: closing flushed file at 1731631879750Flushing 1588230740/ns: creating writer at 1731631879775 (+25 ms)Flushing 1588230740/ns: appending metadata at 1731631879797 (+22 ms)Flushing 1588230740/ns: closing flushed file at 1731631879797Flushing 1588230740/rep_barrier: creating writer at 1731631879811 (+14 ms)Flushing 1588230740/rep_barrier: appending metadata at 1731631879831 (+20 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1731631879831Flushing 1588230740/table: creating writer at 1731631880244 (+413 ms)Flushing 1588230740/table: appending metadata at 1731631880257 (+13 ms)Flushing 1588230740/table: closing flushed file at 1731631880257Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6f0b97f: reopening flushed file at 1731631880664 (+407 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@73c61c97: reopening flushed file at 1731631880669 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58b7f944: reopening flushed file at 1731631880674 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4f4a2fc: reopening flushed file at 1731631880678 (+4 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 966ms, sequenceid=90, compaction requested=false at 1731631880684 (+6 ms)Writing region close event to WAL at 1731631880687 (+3 ms)Running coprocessor post-close hooks at 1731631880690 (+3 ms)Closed at 1731631880690 2024-11-15T00:51:20,690 DEBUG [RS_CLOSE_META-regionserver/fb2f6cb7846a:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-15T00:51:20,719 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(976): stopping server fb2f6cb7846a,42925,1731631698709; all regions closed. 2024-11-15T00:51:20,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741834_1010 (size=25344) 2024-11-15T00:51:20,725 DEBUG [RS:0;fb2f6cb7846a:42925 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/oldWALs 2024-11-15T00:51:20,725 INFO [RS:0;fb2f6cb7846a:42925 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fb2f6cb7846a%2C42925%2C1731631698709.meta:.meta(num 1731631702389) 2024-11-15T00:51:20,727 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741833_1009 (size=14259030) 2024-11-15T00:51:20,729 DEBUG [RS:0;fb2f6cb7846a:42925 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/oldWALs 2024-11-15T00:51:20,729 INFO [RS:0;fb2f6cb7846a:42925 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL fb2f6cb7846a%2C42925%2C1731631698709:(num 1731631701380) 2024-11-15T00:51:20,729 DEBUG [RS:0;fb2f6cb7846a:42925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-15T00:51:20,729 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.LeaseManager(133): Closed leases 2024-11-15T00:51:20,729 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T00:51:20,729 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.ChoreService(370): Chore service for: regionserver/fb2f6cb7846a:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS] on shutdown 2024-11-15T00:51:20,729 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T00:51:20,729 INFO [regionserver/fb2f6cb7846a:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T00:51:20,730 INFO [RS:0;fb2f6cb7846a:42925 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:42925 2024-11-15T00:51:20,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/fb2f6cb7846a,42925,1731631698709 2024-11-15T00:51:20,742 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T00:51:20,742 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-15T00:51:20,756 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [fb2f6cb7846a,42925,1731631698709] 2024-11-15T00:51:20,773 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/fb2f6cb7846a,42925,1731631698709 already deleted, retry=false 2024-11-15T00:51:20,773 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; fb2f6cb7846a,42925,1731631698709 expired; onlineServers=0 2024-11-15T00:51:20,774 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master 'fb2f6cb7846a,46397,1731631697729' ***** 2024-11-15T00:51:20,774 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-15T00:51:20,774 INFO [M:0;fb2f6cb7846a:46397 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-15T00:51:20,774 INFO [M:0;fb2f6cb7846a:46397 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-15T00:51:20,774 DEBUG [M:0;fb2f6cb7846a:46397 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-15T00:51:20,774 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-15T00:51:20,774 DEBUG [M:0;fb2f6cb7846a:46397 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-15T00:51:20,774 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.small.0-1731631700981 {}] cleaner.HFileCleaner(306): Exit Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.small.0-1731631700981,5,FailOnTimeoutGroup] 2024-11-15T00:51:20,774 DEBUG [master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.large.0-1731631700980 {}] cleaner.HFileCleaner(306): Exit Thread[master/fb2f6cb7846a:0:becomeActiveMaster-HFileCleaner.large.0-1731631700980,5,FailOnTimeoutGroup] 2024-11-15T00:51:20,774 INFO [M:0;fb2f6cb7846a:46397 {}] hbase.ChoreService(370): Chore service for: master/fb2f6cb7846a:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-15T00:51:20,775 INFO [M:0;fb2f6cb7846a:46397 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-15T00:51:20,775 DEBUG [M:0;fb2f6cb7846a:46397 {}] master.HMaster(1795): Stopping service threads 2024-11-15T00:51:20,775 INFO [M:0;fb2f6cb7846a:46397 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-15T00:51:20,775 INFO [M:0;fb2f6cb7846a:46397 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-15T00:51:20,775 ERROR [M:0;fb2f6cb7846a:46397 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-15T00:51:20,776 INFO [M:0;fb2f6cb7846a:46397 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-15T00:51:20,776 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-15T00:51:20,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-15T00:51:20,796 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-15T00:51:20,796 DEBUG [M:0;fb2f6cb7846a:46397 {}] zookeeper.ZKUtil(347): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-15T00:51:20,797 WARN [M:0;fb2f6cb7846a:46397 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-15T00:51:20,797 INFO [M:0;fb2f6cb7846a:46397 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/.lastflushedseqids 2024-11-15T00:51:20,807 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742454_1630 (size=72) 2024-11-15T00:51:20,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:51:20,857 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:42925-0x1013bf6c77b0001, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:51:20,857 INFO [RS:0;fb2f6cb7846a:42925 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T00:51:20,858 INFO [RS:0;fb2f6cb7846a:42925 {}] regionserver.HRegionServer(1031): Exiting; stopping=fb2f6cb7846a,42925,1731631698709; zookeeper connection closed. 2024-11-15T00:51:20,859 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@63578ac4 {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@63578ac4 2024-11-15T00:51:20,859 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-15T00:51:21,208 INFO [M:0;fb2f6cb7846a:46397 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-15T00:51:21,208 INFO [M:0;fb2f6cb7846a:46397 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-15T00:51:21,208 DEBUG [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-15T00:51:21,208 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:51:21,208 DEBUG [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:51:21,209 DEBUG [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-15T00:51:21,209 DEBUG [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:51:21,209 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=764.46 KB heapSize=939.48 KB 2024-11-15T00:51:21,229 DEBUG [M:0;fb2f6cb7846a:46397 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/582b7462fff7437d8605398b3ccad4da is 82, key is hbase:meta,,1/info:regioninfo/1731631702482/Put/seqid=0 2024-11-15T00:51:21,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742455_1631 (size=5672) 2024-11-15T00:51:21,249 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2160 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/582b7462fff7437d8605398b3ccad4da 2024-11-15T00:51:21,304 DEBUG [M:0;fb2f6cb7846a:46397 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ae5186415eb94255afbc749f71c3920e is 2277, key is \x00\x00\x00\x00\x00\x00\x00\x93/proc:d/1731631853953/Put/seqid=0 2024-11-15T00:51:21,315 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742456_1632 (size=44555) 2024-11-15T00:51:21,321 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=763.90 KB at sequenceid=2160 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ae5186415eb94255afbc749f71c3920e 2024-11-15T00:51:21,325 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ae5186415eb94255afbc749f71c3920e 2024-11-15T00:51:21,354 DEBUG [M:0;fb2f6cb7846a:46397 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2563e0dbe6354a85bd0538b20865af99 is 69, key is fb2f6cb7846a,42925,1731631698709/rs:state/1731631701035/Put/seqid=0 2024-11-15T00:51:21,365 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073742457_1633 (size=5156) 2024-11-15T00:51:21,766 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2160 (bloomFilter=true), to=hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2563e0dbe6354a85bd0538b20865af99 2024-11-15T00:51:21,772 DEBUG [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/582b7462fff7437d8605398b3ccad4da as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/582b7462fff7437d8605398b3ccad4da 2024-11-15T00:51:21,777 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/582b7462fff7437d8605398b3ccad4da, entries=8, sequenceid=2160, filesize=5.5 K 2024-11-15T00:51:21,778 DEBUG [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/ae5186415eb94255afbc749f71c3920e as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ae5186415eb94255afbc749f71c3920e 2024-11-15T00:51:21,782 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for ae5186415eb94255afbc749f71c3920e 2024-11-15T00:51:21,782 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/ae5186415eb94255afbc749f71c3920e, entries=172, sequenceid=2160, filesize=43.5 K 2024-11-15T00:51:21,782 DEBUG [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/2563e0dbe6354a85bd0538b20865af99 as hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2563e0dbe6354a85bd0538b20865af99 2024-11-15T00:51:21,786 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:34051/user/jenkins/test-data/e3a02ff5-aa44-fca6-6c1a-eecd713b4cdd/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/2563e0dbe6354a85bd0538b20865af99, entries=1, sequenceid=2160, filesize=5.0 K 2024-11-15T00:51:21,787 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegion(3140): Finished flush of dataSize ~764.46 KB/782802, heapSize ~939.19 KB/961728, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 578ms, sequenceid=2160, compaction requested=false 2024-11-15T00:51:21,794 INFO [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-15T00:51:21,794 DEBUG [M:0;fb2f6cb7846a:46397 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1731631881208Disabling compacts and flushes for region at 1731631881208Disabling writes for close at 1731631881209 (+1 ms)Obtaining lock to block concurrent updates at 1731631881209Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1731631881209Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=782802, getHeapSize=961968, getOffHeapSize=0, getCellsCount=2357 at 1731631881209Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1731631881210 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1731631881210Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1731631881229 (+19 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1731631881229Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1731631881254 (+25 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1731631881303 (+49 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1731631881303Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1731631881325 (+22 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1731631881353 (+28 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1731631881353Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@29e9bb9c: reopening flushed file at 1731631881769 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1fde326e: reopening flushed file at 1731631881777 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1ff6586: reopening flushed file at 1731631881782 (+5 ms)Finished flush of dataSize ~764.46 KB/782802, heapSize ~939.19 KB/961728, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 578ms, sequenceid=2160, compaction requested=false at 1731631881787 (+5 ms)Writing region close event to WAL at 1731631881794 (+7 ms)Closed at 1731631881794 2024-11-15T00:51:21,800 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:37847 is added to blk_1073741830_1006 (size=924409) 2024-11-15T00:51:21,801 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-15T00:51:21,801 INFO [M:0;fb2f6cb7846a:46397 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-15T00:51:21,801 INFO [M:0;fb2f6cb7846a:46397 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.3:46397 2024-11-15T00:51:21,801 INFO [M:0;fb2f6cb7846a:46397 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-15T00:51:21,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:51:21,914 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:46397-0x1013bf6c77b0000, quorum=127.0.0.1:59615, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-15T00:51:21,914 INFO [M:0;fb2f6cb7846a:46397 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-15T00:51:21,921 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d1d0948{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-15T00:51:21,924 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T00:51:21,924 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T00:51:21,924 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T00:51:21,924 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/hadoop.log.dir/,STOPPED} 2024-11-15T00:51:21,933 WARN [BP-1791232400-172.17.0.3-1731631693107 heartbeating to localhost/127.0.0.1:34051 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-15T00:51:21,933 WARN [BP-1791232400-172.17.0.3-1731631693107 heartbeating to localhost/127.0.0.1:34051 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-1791232400-172.17.0.3-1731631693107 (Datanode Uuid bfcb1796-17ac-4b3f-8ab9-7fe5e77a4458) service to localhost/127.0.0.1:34051 2024-11-15T00:51:21,934 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-15T00:51:21,934 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-15T00:51:21,937 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/cluster_4d500b26-95f0-bc79-8ccc-862558111e78/data/data1/current/BP-1791232400-172.17.0.3-1731631693107 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T00:51:21,938 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/cluster_4d500b26-95f0-bc79-8ccc-862558111e78/data/data2/current/BP-1791232400-172.17.0.3-1731631693107 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-15T00:51:21,940 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-15T00:51:21,955 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-15T00:51:21,956 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-15T00:51:21,956 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-15T00:51:21,956 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-15T00:51:21,956 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/36c79245-16c0-3839-bc94-5a14c744efeb/hadoop.log.dir/,STOPPED} 2024-11-15T00:51:21,979 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-15T00:51:22,196 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down